drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  */
  19
  20 #include <linux/init.h>
  21 #include <linux/bitmap.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/export.h>
  24 #include <linux/slab.h>
  25 #include <linux/irq.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/pci.h>
  29 #include <linux/dmar.h>
  30 #include <linux/dma-mapping.h>
  31 #include <linux/mempool.h>
  32 #include <linux/memory.h>
  33 #include <linux/timer.h>
  34 #include <linux/iova.h>
  35 #include <linux/iommu.h>
  36 #include <linux/intel-iommu.h>
  37 #include <linux/syscore_ops.h>
  38 #include <linux/tboot.h>
  39 #include <linux/dmi.h>
  40 #include <linux/pci-ats.h>
  41 #include <linux/memblock.h>
  42 #include <linux/dma-contiguous.h>
  43 #include <asm/irq_remapping.h>
  44 #include <asm/cacheflush.h>
  45 #include <asm/iommu.h>
  46
  47 #include "irq_remapping.h"
  48 #include "pci.h"
  49
  50 #define ROOT_SIZE               VTD_PAGE_SIZE
  51 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  52
  53 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  54 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  55 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  56
  57 #define IOAPIC_RANGE_START      (0xfee00000)
  58 #define IOAPIC_RANGE_END        (0xfeefffff)
  59 #define IOVA_START_ADDR         (0x1000)
  60
  61 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  62
  63 #define MAX_AGAW_WIDTH 64
  64 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  65
  66 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  67 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  68
  69 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  70    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  71 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  72                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  73 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  74
  75 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  76 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  77 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  78
  79 /* page table handling */
  80 #define LEVEL_STRIDE            (9)
  81 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  82
  83 /*
  84  * This bitmap is used to advertise the page sizes our hardware support
  85  * to the IOMMU core, which will then use this information to split
  86  * physically contiguous memory regions it is mapping into page sizes
  87  * that we support.
  88  *
  89  * Traditionally the IOMMU core just handed us the mappings directly,
  90  * after making sure the size is an order of a 4KiB page and that the
  91  * mapping has natural alignment.
  92  *
  93  * To retain this behavior, we currently advertise that we support
  94  * all page sizes that are an order of 4KiB.
  95  *
  96  * If at some point we'd like to utilize the IOMMU core's new behavior,
  97  * we could change this to advertise the real page sizes we support.
  98  */
  99 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 100
 101 static inline int agaw_to_level(int agaw)
 102 {
 103         return agaw + 2;
 104 }
 105
 106 static inline int agaw_to_width(int agaw)
 107 {
 108         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 109 }
 110
 111 static inline int width_to_agaw(int width)
 112 {
 113         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 114 }
 115
 116 static inline unsigned int level_to_offset_bits(int level)
 117 {
 118         return (level - 1) * LEVEL_STRIDE;
 119 }
 120
 121 static inline int pfn_level_offset(unsigned long pfn, int level)
 122 {
 123         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 124 }
 125
 126 static inline unsigned long level_mask(int level)
 127 {
 128         return -1UL << level_to_offset_bits(level);
 129 }
 130
 131 static inline unsigned long level_size(int level)
 132 {
 133         return 1UL << level_to_offset_bits(level);
 134 }
 135
 136 static inline unsigned long align_to_level(unsigned long pfn, int level)
 137 {
 138         return (pfn + level_size(level) - 1) & level_mask(level);
 139 }
 140
 141 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 142 {
 143         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 144 }
 145
 146 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 147    are never going to work. */
 148 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 149 {
 150         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 151 }
 152
 153 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 154 {
 155         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 156 }
 157 static inline unsigned long page_to_dma_pfn(struct page *pg)
 158 {
 159         return mm_to_dma_pfn(page_to_pfn(pg));
 160 }
 161 static inline unsigned long virt_to_dma_pfn(void *p)
 162 {
 163         return page_to_dma_pfn(virt_to_page(p));
 164 }
 165
 166 /* global iommu list, set NULL for ignored DMAR units */
 167 static struct intel_iommu **g_iommus;
 168
 169 static void __init check_tylersburg_isoch(void);
 170 static int rwbf_quirk;
 171
 172 /*
 173  * set to 1 to panic kernel if can't successfully enable VT-d
 174  * (used when kernel is launched w/ TXT)
 175  */
 176 static int force_on = 0;
 177
 178 /*
 179  * 0: Present
 180  * 1-11: Reserved
 181  * 12-63: Context Ptr (12 - (haw-1))
 182  * 64-127: Reserved
 183  */
 184 struct root_entry {
 185         u64     val;
 186         u64     rsvd1;
 187 };
 188 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 189 static inline bool root_present(struct root_entry *root)
 190 {
 191         return (root->val & 1);
 192 }
 193 static inline void set_root_present(struct root_entry *root)
 194 {
 195         root->val |= 1;
 196 }
 197 static inline void set_root_value(struct root_entry *root, unsigned long value)
 198 {
 199         root->val |= value & VTD_PAGE_MASK;
 200 }
 201
 202 static inline struct context_entry *
 203 get_context_addr_from_root(struct root_entry *root)
 204 {
 205         return (struct context_entry *)
 206                 (root_present(root)?phys_to_virt(
 207                 root->val & VTD_PAGE_MASK) :
 208                 NULL);
 209 }
 210
 211 /*
 212  * low 64 bits:
 213  * 0: present
 214  * 1: fault processing disable
 215  * 2-3: translation type
 216  * 12-63: address space root
 217  * high 64 bits:
 218  * 0-2: address width
 219  * 3-6: aval
 220  * 8-23: domain id
 221  */
 222 struct context_entry {
 223         u64 lo;
 224         u64 hi;
 225 };
 226
 227 static inline bool context_present(struct context_entry *context)
 228 {
 229         return (context->lo & 1);
 230 }
 231 static inline void context_set_present(struct context_entry *context)
 232 {
 233         context->lo |= 1;
 234 }
 235
 236 static inline void context_set_fault_enable(struct context_entry *context)
 237 {
 238         context->lo &= (((u64)-1) << 2) | 1;
 239 }
 240
 241 static inline void context_set_translation_type(struct context_entry *context,
 242                                                 unsigned long value)
 243 {
 244         context->lo &= (((u64)-1) << 4) | 3;
 245         context->lo |= (value & 3) << 2;
 246 }
 247
 248 static inline void context_set_address_root(struct context_entry *context,
 249                                             unsigned long value)
 250 {
 251         context->lo |= value & VTD_PAGE_MASK;
 252 }
 253
 254 static inline void context_set_address_width(struct context_entry *context,
 255                                              unsigned long value)
 256 {
 257         context->hi |= value & 7;
 258 }
 259
 260 static inline void context_set_domain_id(struct context_entry *context,
 261                                          unsigned long value)
 262 {
 263         context->hi |= (value & ((1 << 16) - 1)) << 8;
 264 }
 265
 266 static inline void context_clear_entry(struct context_entry *context)
 267 {
 268         context->lo = 0;
 269         context->hi = 0;
 270 }
 271
 272 /*
 273  * 0: readable
 274  * 1: writable
 275  * 2-6: reserved
 276  * 7: super page
 277  * 8-10: available
 278  * 11: snoop behavior
 279  * 12-63: Host physcial address
 280  */
 281 struct dma_pte {
 282         u64 val;
 283 };
 284
 285 static inline void dma_clear_pte(struct dma_pte *pte)
 286 {
 287         pte->val = 0;
 288 }
 289
 290 static inline u64 dma_pte_addr(struct dma_pte *pte)
 291 {
 292 #ifdef CONFIG_64BIT
 293         return pte->val & VTD_PAGE_MASK;
 294 #else
 295         /* Must have a full atomic 64-bit read */
 296         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 297 #endif
 298 }
 299
 300 static inline bool dma_pte_present(struct dma_pte *pte)
 301 {
 302         return (pte->val & 3) != 0;
 303 }
 304
 305 static inline bool dma_pte_superpage(struct dma_pte *pte)
 306 {
 307         return (pte->val & (1 << 7));
 308 }
 309
 310 static inline int first_pte_in_page(struct dma_pte *pte)
 311 {
 312         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 313 }
 314
 315 /*
 316  * This domain is a statically identity mapping domain.
 317  *      1. This domain creats a static 1:1 mapping to all usable memory.
 318  *      2. It maps to each iommu if successful.
 319  *      3. Each iommu mapps to this domain if successful.
 320  */
 321 static struct dmar_domain *si_domain;
 322 static int hw_pass_through = 1;
 323
 324 /* devices under the same p2p bridge are owned in one domain */
 325 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 326
 327 /* domain represents a virtual machine, more than one devices
 328  * across iommus may be owned in one domain, e.g. kvm guest.
 329  */
 330 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 331
 332 /* si_domain contains mulitple devices */
 333 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 2)
 334
 335 /* define the limit of IOMMUs supported in each domain */
 336 #ifdef  CONFIG_X86
 337 # define        IOMMU_UNITS_SUPPORTED   MAX_IO_APICS
 338 #else
 339 # define        IOMMU_UNITS_SUPPORTED   64
 340 #endif
 341
 342 struct dmar_domain {
 343         int     id;                     /* domain id */
 344         int     nid;                    /* node id */
 345         DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
 346                                         /* bitmap of iommus this domain uses*/
 347
 348         struct list_head devices;       /* all devices' list */
 349         struct iova_domain iovad;       /* iova's that belong to this domain */
 350
 351         struct dma_pte  *pgd;           /* virtual address */
 352         int             gaw;            /* max guest address width */
 353
 354         /* adjusted guest address width, 0 is level 2 30-bit */
 355         int             agaw;
 356
 357         int             flags;          /* flags to find out type of domain */
 358
 359         int             iommu_coherency;/* indicate coherency of iommu access */
 360         int             iommu_snooping; /* indicate snooping control feature*/
 361         int             iommu_count;    /* reference count of iommu */
 362         int             iommu_superpage;/* Level of superpages supported:
 363                                            0 == 4KiB (no superpages), 1 == 2MiB,
 364                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 365         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 366         u64             max_addr;       /* maximum mapped address */
 367 };
 368
 369 /* PCI domain-device relationship */
 370 struct device_domain_info {
 371         struct list_head link;  /* link to domain siblings */
 372         struct list_head global; /* link to global list */
 373         u8 bus;                 /* PCI bus number */
 374         u8 devfn;               /* PCI devfn number */
 375         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 376         struct intel_iommu *iommu; /* IOMMU used by this device */
 377         struct dmar_domain *domain; /* pointer to domain */
 378 };
 379
 380 struct dmar_rmrr_unit {
 381         struct list_head list;          /* list of rmrr units   */
 382         struct acpi_dmar_header *hdr;   /* ACPI header          */
 383         u64     base_address;           /* reserved base address*/
 384         u64     end_address;            /* reserved end address */
 385         struct dmar_dev_scope *devices; /* target devices */
 386         int     devices_cnt;            /* target device count */
 387 };
 388
 389 struct dmar_atsr_unit {
 390         struct list_head list;          /* list of ATSR units */
 391         struct acpi_dmar_header *hdr;   /* ACPI header */
 392         struct dmar_dev_scope *devices; /* target devices */
 393         int devices_cnt;                /* target device count */
 394         u8 include_all:1;               /* include all ports */
 395 };
 396
 397 static LIST_HEAD(dmar_atsr_units);
 398 static LIST_HEAD(dmar_rmrr_units);
 399
 400 #define for_each_rmrr_units(rmrr) \
 401         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 402
 403 static void flush_unmaps_timeout(unsigned long data);
 404
 405 static DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 406
 407 #define HIGH_WATER_MARK 250
 408 struct deferred_flush_tables {
 409         int next;
 410         struct iova *iova[HIGH_WATER_MARK];
 411         struct dmar_domain *domain[HIGH_WATER_MARK];
 412         struct page *freelist[HIGH_WATER_MARK];
 413 };
 414
 415 static struct deferred_flush_tables *deferred_flush;
 416
 417 /* bitmap for indexing intel_iommus */
 418 static int g_num_of_iommus;
 419
 420 static DEFINE_SPINLOCK(async_umap_flush_lock);
 421 static LIST_HEAD(unmaps_to_do);
 422
 423 static int timer_on;
 424 static long list_size;
 425
 426 static void domain_exit(struct dmar_domain *domain);
 427 static void domain_remove_dev_info(struct dmar_domain *domain);
 428 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 429                                        struct device *dev);
 430 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
 431                                            struct device *dev);
 432
 433 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 434 int dmar_disabled = 0;
 435 #else
 436 int dmar_disabled = 1;
 437 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 438
 439 int intel_iommu_enabled = 0;
 440 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 441
 442 static int dmar_map_gfx = 1;
 443 static int dmar_forcedac;
 444 static int intel_iommu_strict;
 445 static int intel_iommu_superpage = 1;
 446
 447 int intel_iommu_gfx_mapped;
 448 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 449
 450 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 451 static DEFINE_SPINLOCK(device_domain_lock);
 452 static LIST_HEAD(device_domain_list);
 453
 454 static struct iommu_ops intel_iommu_ops;
 455
 456 static int __init intel_iommu_setup(char *str)
 457 {
 458         if (!str)
 459                 return -EINVAL;
 460         while (*str) {
 461                 if (!strncmp(str, "on", 2)) {
 462                         dmar_disabled = 0;
 463                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 464                 } else if (!strncmp(str, "off", 3)) {
 465                         dmar_disabled = 1;
 466                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 467                 } else if (!strncmp(str, "igfx_off", 8)) {
 468                         dmar_map_gfx = 0;
 469                         printk(KERN_INFO
 470                                 "Intel-IOMMU: disable GFX device mapping\n");
 471                 } else if (!strncmp(str, "forcedac", 8)) {
 472                         printk(KERN_INFO
 473                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 474                         dmar_forcedac = 1;
 475                 } else if (!strncmp(str, "strict", 6)) {
 476                         printk(KERN_INFO
 477                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 478                         intel_iommu_strict = 1;
 479                 } else if (!strncmp(str, "sp_off", 6)) {
 480                         printk(KERN_INFO
 481                                 "Intel-IOMMU: disable supported super page\n");
 482                         intel_iommu_superpage = 0;
 483                 }
 484
 485                 str += strcspn(str, ",");
 486                 while (*str == ',')
 487                         str++;
 488         }
 489         return 0;
 490 }
 491 __setup("intel_iommu=", intel_iommu_setup);
 492
 493 static struct kmem_cache *iommu_domain_cache;
 494 static struct kmem_cache *iommu_devinfo_cache;
 495 static struct kmem_cache *iommu_iova_cache;
 496
 497 static inline void *alloc_pgtable_page(int node)
 498 {
 499         struct page *page;
 500         void *vaddr = NULL;
 501
 502         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 503         if (page)
 504                 vaddr = page_address(page);
 505         return vaddr;
 506 }
 507
 508 static inline void free_pgtable_page(void *vaddr)
 509 {
 510         free_page((unsigned long)vaddr);
 511 }
 512
 513 static inline void *alloc_domain_mem(void)
 514 {
 515         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 516 }
 517
 518 static void free_domain_mem(void *vaddr)
 519 {
 520         kmem_cache_free(iommu_domain_cache, vaddr);
 521 }
 522
 523 static inline void * alloc_devinfo_mem(void)
 524 {
 525         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 526 }
 527
 528 static inline void free_devinfo_mem(void *vaddr)
 529 {
 530         kmem_cache_free(iommu_devinfo_cache, vaddr);
 531 }
 532
 533 struct iova *alloc_iova_mem(void)
 534 {
 535         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 536 }
 537
 538 void free_iova_mem(struct iova *iova)
 539 {
 540         kmem_cache_free(iommu_iova_cache, iova);
 541 }
 542
 543
 544 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 545 {
 546         unsigned long sagaw;
 547         int agaw = -1;
 548
 549         sagaw = cap_sagaw(iommu->cap);
 550         for (agaw = width_to_agaw(max_gaw);
 551              agaw >= 0; agaw--) {
 552                 if (test_bit(agaw, &sagaw))
 553                         break;
 554         }
 555
 556         return agaw;
 557 }
 558
 559 /*
 560  * Calculate max SAGAW for each iommu.
 561  */
 562 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 563 {
 564         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 565 }
 566
 567 /*
 568  * calculate agaw for each iommu.
 569  * "SAGAW" may be different across iommus, use a default agaw, and
 570  * get a supported less agaw for iommus that don't support the default agaw.
 571  */
 572 int iommu_calculate_agaw(struct intel_iommu *iommu)
 573 {
 574         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 575 }
 576
 577 /* This functionin only returns single iommu in a domain */
 578 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 579 {
 580         int iommu_id;
 581
 582         /* si_domain and vm domain should not get here. */
 583         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 584         BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 585
 586         iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
 587         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 588                 return NULL;
 589
 590         return g_iommus[iommu_id];
 591 }
 592
 593 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 594 {
 595         struct dmar_drhd_unit *drhd;
 596         struct intel_iommu *iommu;
 597         int i, found = 0;
 598
 599         domain->iommu_coherency = 1;
 600
 601         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 602                 found = 1;
 603                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 604                         domain->iommu_coherency = 0;
 605                         break;
 606                 }
 607         }
 608         if (found)
 609                 return;
 610
 611         /* No hardware attached; use lowest common denominator */
 612         rcu_read_lock();
 613         for_each_active_iommu(iommu, drhd) {
 614                 if (!ecap_coherent(iommu->ecap)) {
 615                         domain->iommu_coherency = 0;
 616                         break;
 617                 }
 618         }
 619         rcu_read_unlock();
 620 }
 621
 622 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 623 {
 624         int i;
 625
 626         domain->iommu_snooping = 1;
 627
 628         for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
 629                 if (!ecap_sc_support(g_iommus[i]->ecap)) {
 630                         domain->iommu_snooping = 0;
 631                         break;
 632                 }
 633         }
 634 }
 635
 636 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 637 {
 638         struct dmar_drhd_unit *drhd;
 639         struct intel_iommu *iommu = NULL;
 640         int mask = 0xf;
 641
 642         if (!intel_iommu_superpage) {
 643                 domain->iommu_superpage = 0;
 644                 return;
 645         }
 646
 647         /* set iommu_superpage to the smallest common denominator */
 648         rcu_read_lock();
 649         for_each_active_iommu(iommu, drhd) {
 650                 mask &= cap_super_page_val(iommu->cap);
 651                 if (!mask) {
 652                         break;
 653                 }
 654         }
 655         rcu_read_unlock();
 656
 657         domain->iommu_superpage = fls(mask);
 658 }
 659
 660 /* Some capabilities may be different across iommus */
 661 static void domain_update_iommu_cap(struct dmar_domain *domain)
 662 {
 663         domain_update_iommu_coherency(domain);
 664         domain_update_iommu_snooping(domain);
 665         domain_update_iommu_superpage(domain);
 666 }
 667
 668 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 669 {
 670         struct dmar_drhd_unit *drhd = NULL;
 671         struct intel_iommu *iommu;
 672         struct device *tmp;
 673         struct pci_dev *ptmp, *pdev = NULL;
 674         u16 segment;
 675         int i;
 676
 677         if (dev_is_pci(dev)) {
 678                 pdev = to_pci_dev(dev);
 679                 segment = pci_domain_nr(pdev->bus);
 680         } else if (ACPI_COMPANION(dev))
 681                 dev = &ACPI_COMPANION(dev)->dev;
 682
 683         rcu_read_lock();
 684         for_each_active_iommu(iommu, drhd) {
 685                 if (pdev && segment != drhd->segment)
 686                         continue;
 687
 688                 for_each_active_dev_scope(drhd->devices,
 689                                           drhd->devices_cnt, i, tmp) {
 690                         if (tmp == dev) {
 691                                 *bus = drhd->devices[i].bus;
 692                                 *devfn = drhd->devices[i].devfn;
 693                                 goto out;
 694                         }
 695
 696                         if (!pdev || !dev_is_pci(tmp))
 697                                 continue;
 698
 699                         ptmp = to_pci_dev(tmp);
 700                         if (ptmp->subordinate &&
 701                             ptmp->subordinate->number <= pdev->bus->number &&
 702                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 703                                 goto got_pdev;
 704                 }
 705
 706                 if (pdev && drhd->include_all) {
 707                 got_pdev:
 708                         *bus = pdev->bus->number;
 709                         *devfn = pdev->devfn;
 710                         goto out;
 711                 }
 712         }
 713         iommu = NULL;
 714  out:
 715         rcu_read_unlock();
 716
 717         return iommu;
 718 }
 719
 720 static void domain_flush_cache(struct dmar_domain *domain,
 721                                void *addr, int size)
 722 {
 723         if (!domain->iommu_coherency)
 724                 clflush_cache_range(addr, size);
 725 }
 726
 727 /* Gets context entry for a given bus and devfn */
 728 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 729                 u8 bus, u8 devfn)
 730 {
 731         struct root_entry *root;
 732         struct context_entry *context;
 733         unsigned long phy_addr;
 734         unsigned long flags;
 735
 736         spin_lock_irqsave(&iommu->lock, flags);
 737         root = &iommu->root_entry[bus];
 738         context = get_context_addr_from_root(root);
 739         if (!context) {
 740                 context = (struct context_entry *)
 741                                 alloc_pgtable_page(iommu->node);
 742                 if (!context) {
 743                         spin_unlock_irqrestore(&iommu->lock, flags);
 744                         return NULL;
 745                 }
 746                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 747                 phy_addr = virt_to_phys((void *)context);
 748                 set_root_value(root, phy_addr);
 749                 set_root_present(root);
 750                 __iommu_flush_cache(iommu, root, sizeof(*root));
 751         }
 752         spin_unlock_irqrestore(&iommu->lock, flags);
 753         return &context[devfn];
 754 }
 755
 756 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 757 {
 758         struct root_entry *root;
 759         struct context_entry *context;
 760         int ret;
 761         unsigned long flags;
 762
 763         spin_lock_irqsave(&iommu->lock, flags);
 764         root = &iommu->root_entry[bus];
 765         context = get_context_addr_from_root(root);
 766         if (!context) {
 767                 ret = 0;
 768                 goto out;
 769         }
 770         ret = context_present(&context[devfn]);
 771 out:
 772         spin_unlock_irqrestore(&iommu->lock, flags);
 773         return ret;
 774 }
 775
 776 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 777 {
 778         struct root_entry *root;
 779         struct context_entry *context;
 780         unsigned long flags;
 781
 782         spin_lock_irqsave(&iommu->lock, flags);
 783         root = &iommu->root_entry[bus];
 784         context = get_context_addr_from_root(root);
 785         if (context) {
 786                 context_clear_entry(&context[devfn]);
 787                 __iommu_flush_cache(iommu, &context[devfn], \
 788                         sizeof(*context));
 789         }
 790         spin_unlock_irqrestore(&iommu->lock, flags);
 791 }
 792
 793 static void free_context_table(struct intel_iommu *iommu)
 794 {
 795         struct root_entry *root;
 796         int i;
 797         unsigned long flags;
 798         struct context_entry *context;
 799
 800         spin_lock_irqsave(&iommu->lock, flags);
 801         if (!iommu->root_entry) {
 802                 goto out;
 803         }
 804         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 805                 root = &iommu->root_entry[i];
 806                 context = get_context_addr_from_root(root);
 807                 if (context)
 808                         free_pgtable_page(context);
 809         }
 810         free_pgtable_page(iommu->root_entry);
 811         iommu->root_entry = NULL;
 812 out:
 813         spin_unlock_irqrestore(&iommu->lock, flags);
 814 }
 815
 816 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 817                                       unsigned long pfn, int *target_level)
 818 {
 819         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 820         struct dma_pte *parent, *pte = NULL;
 821         int level = agaw_to_level(domain->agaw);
 822         int offset;
 823
 824         BUG_ON(!domain->pgd);
 825
 826         if (addr_width < BITS_PER_LONG && pfn >> addr_width)
 827                 /* Address beyond IOMMU's addressing capabilities. */
 828                 return NULL;
 829
 830         parent = domain->pgd;
 831
 832         while (1) {
 833                 void *tmp_page;
 834
 835                 offset = pfn_level_offset(pfn, level);
 836                 pte = &parent[offset];
 837                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 838                         break;
 839                 if (level == *target_level)
 840                         break;
 841
 842                 if (!dma_pte_present(pte)) {
 843                         uint64_t pteval;
 844
 845                         tmp_page = alloc_pgtable_page(domain->nid);
 846
 847                         if (!tmp_page)
 848                                 return NULL;
 849
 850                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 851                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 852                         if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 853                                 /* Someone else set it while we were thinking; use theirs. */
 854                                 free_pgtable_page(tmp_page);
 855                         } else {
 856                                 dma_pte_addr(pte);
 857                                 domain_flush_cache(domain, pte, sizeof(*pte));
 858                         }
 859                 }
 860                 if (level == 1)
 861                         break;
 862
 863                 parent = phys_to_virt(dma_pte_addr(pte));
 864                 level--;
 865         }
 866
 867         if (!*target_level)
 868                 *target_level = level;
 869
 870         return pte;
 871 }
 872
 873
 874 /* return address's pte at specific level */
 875 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 876                                          unsigned long pfn,
 877                                          int level, int *large_page)
 878 {
 879         struct dma_pte *parent, *pte = NULL;
 880         int total = agaw_to_level(domain->agaw);
 881         int offset;
 882
 883         parent = domain->pgd;
 884         while (level <= total) {
 885                 offset = pfn_level_offset(pfn, total);
 886                 pte = &parent[offset];
 887                 if (level == total)
 888                         return pte;
 889
 890                 if (!dma_pte_present(pte)) {
 891                         *large_page = total;
 892                         break;
 893                 }
 894
 895                 if (dma_pte_superpage(pte)) {
 896                         *large_page = total;
 897                         return pte;
 898                 }
 899
 900                 parent = phys_to_virt(dma_pte_addr(pte));
 901                 total--;
 902         }
 903         return NULL;
 904 }
 905
 906 /* clear last level pte, a tlb flush should be followed */
 907 static void dma_pte_clear_range(struct dmar_domain *domain,
 908                                 unsigned long start_pfn,
 909                                 unsigned long last_pfn)
 910 {
 911         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 912         unsigned int large_page = 1;
 913         struct dma_pte *first_pte, *pte;
 914
 915         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 916         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 917         BUG_ON(start_pfn > last_pfn);
 918
 919         /* we don't need lock here; nobody else touches the iova range */
 920         do {
 921                 large_page = 1;
 922                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 923                 if (!pte) {
 924                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 925                         continue;
 926                 }
 927                 do {
 928                         dma_clear_pte(pte);
 929                         start_pfn += lvl_to_nr_pages(large_page);
 930                         pte++;
 931                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 932
 933                 domain_flush_cache(domain, first_pte,
 934                                    (void *)pte - (void *)first_pte);
 935
 936         } while (start_pfn && start_pfn <= last_pfn);
 937 }
 938
 939 static void dma_pte_free_level(struct dmar_domain *domain, int level,
 940                                struct dma_pte *pte, unsigned long pfn,
 941                                unsigned long start_pfn, unsigned long last_pfn)
 942 {
 943         pfn = max(start_pfn, pfn);
 944         pte = &pte[pfn_level_offset(pfn, level)];
 945
 946         do {
 947                 unsigned long level_pfn;
 948                 struct dma_pte *level_pte;
 949
 950                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
 951                         goto next;
 952
 953                 level_pfn = pfn & level_mask(level - 1);
 954                 level_pte = phys_to_virt(dma_pte_addr(pte));
 955
 956                 if (level > 2)
 957                         dma_pte_free_level(domain, level - 1, level_pte,
 958                                            level_pfn, start_pfn, last_pfn);
 959
 960                 /* If range covers entire pagetable, free it */
 961                 if (!(start_pfn > level_pfn ||
 962                       last_pfn < level_pfn + level_size(level) - 1)) {
 963                         dma_clear_pte(pte);
 964                         domain_flush_cache(domain, pte, sizeof(*pte));
 965                         free_pgtable_page(level_pte);
 966                 }
 967 next:
 968                 pfn += level_size(level);
 969         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
 970 }
 971
 972 /* free page table pages. last level pte should already be cleared */
 973 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 974                                    unsigned long start_pfn,
 975                                    unsigned long last_pfn)
 976 {
 977         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 978
 979         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 980         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 981         BUG_ON(start_pfn > last_pfn);
 982
 983         /* We don't need lock here; nobody else touches the iova range */
 984         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
 985                            domain->pgd, 0, start_pfn, last_pfn);
 986
 987         /* free pgd */
 988         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 989                 free_pgtable_page(domain->pgd);
 990                 domain->pgd = NULL;
 991         }
 992 }
 993
 994 /* When a page at a given level is being unlinked from its parent, we don't
 995    need to *modify* it at all. All we need to do is make a list of all the
 996    pages which can be freed just as soon as we've flushed the IOTLB and we
 997    know the hardware page-walk will no longer touch them.
 998    The 'pte' argument is the *parent* PTE, pointing to the page that is to
 999    be freed. */
1000 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1001                                             int level, struct dma_pte *pte,
1002                                             struct page *freelist)
1003 {
1004         struct page *pg;
1005
1006         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1007         pg->freelist = freelist;
1008         freelist = pg;
1009
1010         if (level == 1)
1011                 return freelist;
1012
1013         pte = page_address(pg);
1014         do {
1015                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1016                         freelist = dma_pte_list_pagetables(domain, level - 1,
1017                                                            pte, freelist);
1018                 pte++;
1019         } while (!first_pte_in_page(pte));
1020
1021         return freelist;
1022 }
1023
1024 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1025                                         struct dma_pte *pte, unsigned long pfn,
1026                                         unsigned long start_pfn,
1027                                         unsigned long last_pfn,
1028                                         struct page *freelist)
1029 {
1030         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1031
1032         pfn = max(start_pfn, pfn);
1033         pte = &pte[pfn_level_offset(pfn, level)];
1034
1035         do {
1036                 unsigned long level_pfn;
1037
1038                 if (!dma_pte_present(pte))
1039                         goto next;
1040
1041                 level_pfn = pfn & level_mask(level);
1042
1043                 /* If range covers entire pagetable, free it */
1044                 if (start_pfn <= level_pfn &&
1045                     last_pfn >= level_pfn + level_size(level) - 1) {
1046                         /* These suborbinate page tables are going away entirely. Don't
1047                            bother to clear them; we're just going to *free* them. */
1048                         if (level > 1 && !dma_pte_superpage(pte))
1049                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1050
1051                         dma_clear_pte(pte);
1052                         if (!first_pte)
1053                                 first_pte = pte;
1054                         last_pte = pte;
1055                 } else if (level > 1) {
1056                         /* Recurse down into a level that isn't *entirely* obsolete */
1057                         freelist = dma_pte_clear_level(domain, level - 1,
1058                                                        phys_to_virt(dma_pte_addr(pte)),
1059                                                        level_pfn, start_pfn, last_pfn,
1060                                                        freelist);
1061                 }
1062 next:
1063                 pfn += level_size(level);
1064         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1065
1066         if (first_pte)
1067                 domain_flush_cache(domain, first_pte,
1068                                    (void *)++last_pte - (void *)first_pte);
1069
1070         return freelist;
1071 }
1072
1073 /* We can't just free the pages because the IOMMU may still be walking
1074    the page tables, and may have cached the intermediate levels. The
1075    pages can only be freed after the IOTLB flush has been done. */
1076 struct page *domain_unmap(struct dmar_domain *domain,
1077                           unsigned long start_pfn,
1078                           unsigned long last_pfn)
1079 {
1080         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1081         struct page *freelist = NULL;
1082
1083         BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1084         BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1085         BUG_ON(start_pfn > last_pfn);
1086
1087         /* we don't need lock here; nobody else touches the iova range */
1088         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1089                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1090
1091         /* free pgd */
1092         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1093                 struct page *pgd_page = virt_to_page(domain->pgd);
1094                 pgd_page->freelist = freelist;
1095                 freelist = pgd_page;
1096
1097                 domain->pgd = NULL;
1098         }
1099
1100         return freelist;
1101 }
1102
1103 void dma_free_pagelist(struct page *freelist)
1104 {
1105         struct page *pg;
1106
1107         while ((pg = freelist)) {
1108                 freelist = pg->freelist;
1109                 free_pgtable_page(page_address(pg));
1110         }
1111 }
1112
1113 /* iommu handling */
1114 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1115 {
1116         struct root_entry *root;
1117         unsigned long flags;
1118
1119         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1120         if (!root)
1121                 return -ENOMEM;
1122
1123         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1124
1125         spin_lock_irqsave(&iommu->lock, flags);
1126         iommu->root_entry = root;
1127         spin_unlock_irqrestore(&iommu->lock, flags);
1128
1129         return 0;
1130 }
1131
1132 static void iommu_set_root_entry(struct intel_iommu *iommu)
1133 {
1134         void *addr;
1135         u32 sts;
1136         unsigned long flag;
1137
1138         addr = iommu->root_entry;
1139
1140         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1141         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1142
1143         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1144
1145         /* Make sure hardware complete it */
1146         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1147                       readl, (sts & DMA_GSTS_RTPS), sts);
1148
1149         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1150 }
1151
1152 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1153 {
1154         u32 val;
1155         unsigned long flag;
1156
1157         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1158                 return;
1159
1160         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1161         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1162
1163         /* Make sure hardware complete it */
1164         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1165                       readl, (!(val & DMA_GSTS_WBFS)), val);
1166
1167         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1168 }
1169
1170 /* return value determine if we need a write buffer flush */
1171 static void __iommu_flush_context(struct intel_iommu *iommu,
1172                                   u16 did, u16 source_id, u8 function_mask,
1173                                   u64 type)
1174 {
1175         u64 val = 0;
1176         unsigned long flag;
1177
1178         switch (type) {
1179         case DMA_CCMD_GLOBAL_INVL:
1180                 val = DMA_CCMD_GLOBAL_INVL;
1181                 break;
1182         case DMA_CCMD_DOMAIN_INVL:
1183                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1184                 break;
1185         case DMA_CCMD_DEVICE_INVL:
1186                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1187                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1188                 break;
1189         default:
1190                 BUG();
1191         }
1192         val |= DMA_CCMD_ICC;
1193
1194         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1195         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1196
1197         /* Make sure hardware complete it */
1198         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1199                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1200
1201         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1202 }
1203
1204 /* return value determine if we need a write buffer flush */
1205 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1206                                 u64 addr, unsigned int size_order, u64 type)
1207 {
1208         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1209         u64 val = 0, val_iva = 0;
1210         unsigned long flag;
1211
1212         switch (type) {
1213         case DMA_TLB_GLOBAL_FLUSH:
1214                 /* global flush doesn't need set IVA_REG */
1215                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1216                 break;
1217         case DMA_TLB_DSI_FLUSH:
1218                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1219                 break;
1220         case DMA_TLB_PSI_FLUSH:
1221                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1222                 /* IH bit is passed in as part of address */
1223                 val_iva = size_order | addr;
1224                 break;
1225         default:
1226                 BUG();
1227         }
1228         /* Note: set drain read/write */
1229 #if 0
1230         /*
1231          * This is probably to be super secure.. Looks like we can
1232          * ignore it without any impact.
1233          */
1234         if (cap_read_drain(iommu->cap))
1235                 val |= DMA_TLB_READ_DRAIN;
1236 #endif
1237         if (cap_write_drain(iommu->cap))
1238                 val |= DMA_TLB_WRITE_DRAIN;
1239
1240         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1241         /* Note: Only uses first TLB reg currently */
1242         if (val_iva)
1243                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1244         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1245
1246         /* Make sure hardware complete it */
1247         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1248                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1249
1250         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1251
1252         /* check IOTLB invalidation granularity */
1253         if (DMA_TLB_IAIG(val) == 0)
1254                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1255         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1256                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1257                         (unsigned long long)DMA_TLB_IIRG(type),
1258                         (unsigned long long)DMA_TLB_IAIG(val));
1259 }
1260
1261 static struct device_domain_info *
1262 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1263                          u8 bus, u8 devfn)
1264 {
1265         int found = 0;
1266         unsigned long flags;
1267         struct device_domain_info *info;
1268         struct pci_dev *pdev;
1269
1270         if (!ecap_dev_iotlb_support(iommu->ecap))
1271                 return NULL;
1272
1273         if (!iommu->qi)
1274                 return NULL;
1275
1276         spin_lock_irqsave(&device_domain_lock, flags);
1277         list_for_each_entry(info, &domain->devices, link)
1278                 if (info->bus == bus && info->devfn == devfn) {
1279                         found = 1;
1280                         break;
1281                 }
1282         spin_unlock_irqrestore(&device_domain_lock, flags);
1283
1284         if (!found || !info->dev || !dev_is_pci(info->dev))
1285                 return NULL;
1286
1287         pdev = to_pci_dev(info->dev);
1288
1289         if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1290                 return NULL;
1291
1292         if (!dmar_find_matched_atsr_unit(pdev))
1293                 return NULL;
1294
1295         return info;
1296 }
1297
1298 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1299 {
1300         if (!info || !dev_is_pci(info->dev))
1301                 return;
1302
1303         pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1304 }
1305
1306 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1307 {
1308         if (!info->dev || !dev_is_pci(info->dev) ||
1309             !pci_ats_enabled(to_pci_dev(info->dev)))
1310                 return;
1311
1312         pci_disable_ats(to_pci_dev(info->dev));
1313 }
1314
1315 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1316                                   u64 addr, unsigned mask)
1317 {
1318         u16 sid, qdep;
1319         unsigned long flags;
1320         struct device_domain_info *info;
1321
1322         spin_lock_irqsave(&device_domain_lock, flags);
1323         list_for_each_entry(info, &domain->devices, link) {
1324                 struct pci_dev *pdev;
1325                 if (!info->dev || !dev_is_pci(info->dev))
1326                         continue;
1327
1328                 pdev = to_pci_dev(info->dev);
1329                 if (!pci_ats_enabled(pdev))
1330                         continue;
1331
1332                 sid = info->bus << 8 | info->devfn;
1333                 qdep = pci_ats_queue_depth(pdev);
1334                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1335         }
1336         spin_unlock_irqrestore(&device_domain_lock, flags);
1337 }
1338
1339 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1340                                   unsigned long pfn, unsigned int pages, int ih, int map)
1341 {
1342         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1343         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1344
1345         BUG_ON(pages == 0);
1346
1347         if (ih)
1348                 ih = 1 << 6;
1349         /*
1350          * Fallback to domain selective flush if no PSI support or the size is
1351          * too big.
1352          * PSI requires page size to be 2 ^ x, and the base address is naturally
1353          * aligned to the size
1354          */
1355         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1356                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1357                                                 DMA_TLB_DSI_FLUSH);
1358         else
1359                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1360                                                 DMA_TLB_PSI_FLUSH);
1361
1362         /*
1363          * In caching mode, changes of pages from non-present to present require
1364          * flush. However, device IOTLB doesn't need to be flushed in this case.
1365          */
1366         if (!cap_caching_mode(iommu->cap) || !map)
1367                 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1368 }
1369
1370 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1371 {
1372         u32 pmen;
1373         unsigned long flags;
1374
1375         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1376         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1377         pmen &= ~DMA_PMEN_EPM;
1378         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1379
1380         /* wait for the protected region status bit to clear */
1381         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1382                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1383
1384         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1385 }
1386
1387 static int iommu_enable_translation(struct intel_iommu *iommu)
1388 {
1389         u32 sts;
1390         unsigned long flags;
1391
1392         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1393         iommu->gcmd |= DMA_GCMD_TE;
1394         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1395
1396         /* Make sure hardware complete it */
1397         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1398                       readl, (sts & DMA_GSTS_TES), sts);
1399
1400         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1401         return 0;
1402 }
1403
1404 static int iommu_disable_translation(struct intel_iommu *iommu)
1405 {
1406         u32 sts;
1407         unsigned long flag;
1408
1409         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1410         iommu->gcmd &= ~DMA_GCMD_TE;
1411         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1412
1413         /* Make sure hardware complete it */
1414         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1415                       readl, (!(sts & DMA_GSTS_TES)), sts);
1416
1417         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1418         return 0;
1419 }
1420
1421
1422 static int iommu_init_domains(struct intel_iommu *iommu)
1423 {
1424         unsigned long ndomains;
1425         unsigned long nlongs;
1426
1427         ndomains = cap_ndoms(iommu->cap);
1428         pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1429                  iommu->seq_id, ndomains);
1430         nlongs = BITS_TO_LONGS(ndomains);
1431
1432         spin_lock_init(&iommu->lock);
1433
1434         /* TBD: there might be 64K domains,
1435          * consider other allocation for future chip
1436          */
1437         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1438         if (!iommu->domain_ids) {
1439                 pr_err("IOMMU%d: allocating domain id array failed\n",
1440                        iommu->seq_id);
1441                 return -ENOMEM;
1442         }
1443         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1444                         GFP_KERNEL);
1445         if (!iommu->domains) {
1446                 pr_err("IOMMU%d: allocating domain array failed\n",
1447                        iommu->seq_id);
1448                 kfree(iommu->domain_ids);
1449                 iommu->domain_ids = NULL;
1450                 return -ENOMEM;
1451         }
1452
1453         /*
1454          * if Caching mode is set, then invalid translations are tagged
1455          * with domainid 0. Hence we need to pre-allocate it.
1456          */
1457         if (cap_caching_mode(iommu->cap))
1458                 set_bit(0, iommu->domain_ids);
1459         return 0;
1460 }
1461
1462 static void free_dmar_iommu(struct intel_iommu *iommu)
1463 {
1464         struct dmar_domain *domain;
1465         int i, count;
1466         unsigned long flags;
1467
1468         if ((iommu->domains) && (iommu->domain_ids)) {
1469                 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1470                         /*
1471                          * Domain id 0 is reserved for invalid translation
1472                          * if hardware supports caching mode.
1473                          */
1474                         if (cap_caching_mode(iommu->cap) && i == 0)
1475                                 continue;
1476
1477                         domain = iommu->domains[i];
1478                         clear_bit(i, iommu->domain_ids);
1479
1480                         spin_lock_irqsave(&domain->iommu_lock, flags);
1481                         count = --domain->iommu_count;
1482                         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1483                         if (count == 0)
1484                                 domain_exit(domain);
1485                 }
1486         }
1487
1488         if (iommu->gcmd & DMA_GCMD_TE)
1489                 iommu_disable_translation(iommu);
1490
1491         kfree(iommu->domains);
1492         kfree(iommu->domain_ids);
1493         iommu->domains = NULL;
1494         iommu->domain_ids = NULL;
1495
1496         g_iommus[iommu->seq_id] = NULL;
1497
1498         /* free context mapping */
1499         free_context_table(iommu);
1500 }
1501
1502 static struct dmar_domain *alloc_domain(bool vm)
1503 {
1504         /* domain id for virtual machine, it won't be set in context */
1505         static atomic_t vm_domid = ATOMIC_INIT(0);
1506         struct dmar_domain *domain;
1507
1508         domain = alloc_domain_mem();
1509         if (!domain)
1510                 return NULL;
1511
1512         domain->nid = -1;
1513         domain->iommu_count = 0;
1514         memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1515         domain->flags = 0;
1516         spin_lock_init(&domain->iommu_lock);
1517         INIT_LIST_HEAD(&domain->devices);
1518         if (vm) {
1519                 domain->id = atomic_inc_return(&vm_domid);
1520                 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1521         }
1522
1523         return domain;
1524 }
1525
1526 static int iommu_attach_domain(struct dmar_domain *domain,
1527                                struct intel_iommu *iommu)
1528 {
1529         int num;
1530         unsigned long ndomains;
1531         unsigned long flags;
1532
1533         ndomains = cap_ndoms(iommu->cap);
1534
1535         spin_lock_irqsave(&iommu->lock, flags);
1536
1537         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1538         if (num >= ndomains) {
1539                 spin_unlock_irqrestore(&iommu->lock, flags);
1540                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1541                 return -ENOMEM;
1542         }
1543
1544         domain->id = num;
1545         domain->iommu_count++;
1546         set_bit(num, iommu->domain_ids);
1547         set_bit(iommu->seq_id, domain->iommu_bmp);
1548         iommu->domains[num] = domain;
1549         spin_unlock_irqrestore(&iommu->lock, flags);
1550
1551         return 0;
1552 }
1553
1554 static void iommu_detach_domain(struct dmar_domain *domain,
1555                                 struct intel_iommu *iommu)
1556 {
1557         unsigned long flags;
1558         int num, ndomains;
1559
1560         spin_lock_irqsave(&iommu->lock, flags);
1561         ndomains = cap_ndoms(iommu->cap);
1562         for_each_set_bit(num, iommu->domain_ids, ndomains) {
1563                 if (iommu->domains[num] == domain) {
1564                         clear_bit(num, iommu->domain_ids);
1565                         iommu->domains[num] = NULL;
1566                         break;
1567                 }
1568         }
1569         spin_unlock_irqrestore(&iommu->lock, flags);
1570 }
1571
1572 static struct iova_domain reserved_iova_list;
1573 static struct lock_class_key reserved_rbtree_key;
1574
1575 static int dmar_init_reserved_ranges(void)
1576 {
1577         struct pci_dev *pdev = NULL;
1578         struct iova *iova;
1579         int i;
1580
1581         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1582
1583         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1584                 &reserved_rbtree_key);
1585
1586         /* IOAPIC ranges shouldn't be accessed by DMA */
1587         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1588                 IOVA_PFN(IOAPIC_RANGE_END));
1589         if (!iova) {
1590                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1591                 return -ENODEV;
1592         }
1593
1594         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1595         for_each_pci_dev(pdev) {
1596                 struct resource *r;
1597
1598                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1599                         r = &pdev->resource[i];
1600                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1601                                 continue;
1602                         iova = reserve_iova(&reserved_iova_list,
1603                                             IOVA_PFN(r->start),
1604                                             IOVA_PFN(r->end));
1605                         if (!iova) {
1606                                 printk(KERN_ERR "Reserve iova failed\n");
1607                                 return -ENODEV;
1608                         }
1609                 }
1610         }
1611         return 0;
1612 }
1613
1614 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1615 {
1616         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1617 }
1618
1619 static inline int guestwidth_to_adjustwidth(int gaw)
1620 {
1621         int agaw;
1622         int r = (gaw - 12) % 9;
1623
1624         if (r == 0)
1625                 agaw = gaw;
1626         else
1627                 agaw = gaw + 9 - r;
1628         if (agaw > 64)
1629                 agaw = 64;
1630         return agaw;
1631 }
1632
1633 static int domain_init(struct dmar_domain *domain, int guest_width)
1634 {
1635         struct intel_iommu *iommu;
1636         int adjust_width, agaw;
1637         unsigned long sagaw;
1638
1639         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1640         domain_reserve_special_ranges(domain);
1641
1642         /* calculate AGAW */
1643         iommu = domain_get_iommu(domain);
1644         if (guest_width > cap_mgaw(iommu->cap))
1645                 guest_width = cap_mgaw(iommu->cap);
1646         domain->gaw = guest_width;
1647         adjust_width = guestwidth_to_adjustwidth(guest_width);
1648         agaw = width_to_agaw(adjust_width);
1649         sagaw = cap_sagaw(iommu->cap);
1650         if (!test_bit(agaw, &sagaw)) {
1651                 /* hardware doesn't support it, choose a bigger one */
1652                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1653                 agaw = find_next_bit(&sagaw, 5, agaw);
1654                 if (agaw >= 5)
1655                         return -ENODEV;
1656         }
1657         domain->agaw = agaw;
1658
1659         if (ecap_coherent(iommu->ecap))
1660                 domain->iommu_coherency = 1;
1661         else
1662                 domain->iommu_coherency = 0;
1663
1664         if (ecap_sc_support(iommu->ecap))
1665                 domain->iommu_snooping = 1;
1666         else
1667                 domain->iommu_snooping = 0;
1668
1669         if (intel_iommu_superpage)
1670                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1671         else
1672                 domain->iommu_superpage = 0;
1673
1674         domain->nid = iommu->node;
1675
1676         /* always allocate the top pgd */
1677         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1678         if (!domain->pgd)
1679                 return -ENOMEM;
1680         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1681         return 0;
1682 }
1683
1684 static void domain_exit(struct dmar_domain *domain)
1685 {
1686         struct dmar_drhd_unit *drhd;
1687         struct intel_iommu *iommu;
1688         struct page *freelist = NULL;
1689
1690         /* Domain 0 is reserved, so dont process it */
1691         if (!domain)
1692                 return;
1693
1694         /* Flush any lazy unmaps that may reference this domain */
1695         if (!intel_iommu_strict)
1696                 flush_unmaps_timeout(0);
1697
1698         /* remove associated devices */
1699         domain_remove_dev_info(domain);
1700
1701         /* destroy iovas */
1702         put_iova_domain(&domain->iovad);
1703
1704         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1705
1706         /* clear attached or cached domains */
1707         rcu_read_lock();
1708         for_each_active_iommu(iommu, drhd)
1709                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1710                     test_bit(iommu->seq_id, domain->iommu_bmp))
1711                         iommu_detach_domain(domain, iommu);
1712         rcu_read_unlock();
1713
1714         dma_free_pagelist(freelist);
1715
1716         free_domain_mem(domain);
1717 }
1718
1719 static int domain_context_mapping_one(struct dmar_domain *domain,
1720                                       struct intel_iommu *iommu,
1721                                       u8 bus, u8 devfn, int translation)
1722 {
1723         struct context_entry *context;
1724         unsigned long flags;
1725         struct dma_pte *pgd;
1726         unsigned long num;
1727         unsigned long ndomains;
1728         int id;
1729         int agaw;
1730         struct device_domain_info *info = NULL;
1731
1732         pr_debug("Set context mapping for %02x:%02x.%d\n",
1733                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1734
1735         BUG_ON(!domain->pgd);
1736         BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1737                translation != CONTEXT_TT_MULTI_LEVEL);
1738
1739         context = device_to_context_entry(iommu, bus, devfn);
1740         if (!context)
1741                 return -ENOMEM;
1742         spin_lock_irqsave(&iommu->lock, flags);
1743         if (context_present(context)) {
1744                 spin_unlock_irqrestore(&iommu->lock, flags);
1745                 return 0;
1746         }
1747
1748         id = domain->id;
1749         pgd = domain->pgd;
1750
1751         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1752             domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1753                 int found = 0;
1754
1755                 /* find an available domain id for this device in iommu */
1756                 ndomains = cap_ndoms(iommu->cap);
1757                 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1758                         if (iommu->domains[num] == domain) {
1759                                 id = num;
1760                                 found = 1;
1761                                 break;
1762                         }
1763                 }
1764
1765                 if (found == 0) {
1766                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1767                         if (num >= ndomains) {
1768                                 spin_unlock_irqrestore(&iommu->lock, flags);
1769                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1770                                 return -EFAULT;
1771                         }
1772
1773                         set_bit(num, iommu->domain_ids);
1774                         iommu->domains[num] = domain;
1775                         id = num;
1776                 }
1777
1778                 /* Skip top levels of page tables for
1779                  * iommu which has less agaw than default.
1780                  * Unnecessary for PT mode.
1781                  */
1782                 if (translation != CONTEXT_TT_PASS_THROUGH) {
1783                         for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1784                                 pgd = phys_to_virt(dma_pte_addr(pgd));
1785                                 if (!dma_pte_present(pgd)) {
1786                                         spin_unlock_irqrestore(&iommu->lock, flags);
1787                                         return -ENOMEM;
1788                                 }
1789                         }
1790                 }
1791         }
1792
1793         context_set_domain_id(context, id);
1794
1795         if (translation != CONTEXT_TT_PASS_THROUGH) {
1796                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1797                 translation = info ? CONTEXT_TT_DEV_IOTLB :
1798                                      CONTEXT_TT_MULTI_LEVEL;
1799         }
1800         /*
1801          * In pass through mode, AW must be programmed to indicate the largest
1802          * AGAW value supported by hardware. And ASR is ignored by hardware.
1803          */
1804         if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1805                 context_set_address_width(context, iommu->msagaw);
1806         else {
1807                 context_set_address_root(context, virt_to_phys(pgd));
1808                 context_set_address_width(context, iommu->agaw);
1809         }
1810
1811         context_set_translation_type(context, translation);
1812         context_set_fault_enable(context);
1813         context_set_present(context);
1814         domain_flush_cache(domain, context, sizeof(*context));
1815
1816         /*
1817          * It's a non-present to present mapping. If hardware doesn't cache
1818          * non-present entry we only need to flush the write-buffer. If the
1819          * _does_ cache non-present entries, then it does so in the special
1820          * domain #0, which we have to flush:
1821          */
1822         if (cap_caching_mode(iommu->cap)) {
1823                 iommu->flush.flush_context(iommu, 0,
1824                                            (((u16)bus) << 8) | devfn,
1825                                            DMA_CCMD_MASK_NOBIT,
1826                                            DMA_CCMD_DEVICE_INVL);
1827                 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1828         } else {
1829                 iommu_flush_write_buffer(iommu);
1830         }
1831         iommu_enable_dev_iotlb(info);
1832         spin_unlock_irqrestore(&iommu->lock, flags);
1833
1834         spin_lock_irqsave(&domain->iommu_lock, flags);
1835         if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1836                 domain->iommu_count++;
1837                 if (domain->iommu_count == 1)
1838                         domain->nid = iommu->node;
1839                 domain_update_iommu_cap(domain);
1840         }
1841         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1842         return 0;
1843 }
1844
1845 static int
1846 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1847                        int translation)
1848 {
1849         int ret;
1850         struct pci_dev *pdev, *tmp, *parent;
1851         struct intel_iommu *iommu;
1852         u8 bus, devfn;
1853
1854         iommu = device_to_iommu(dev, &bus, &devfn);
1855         if (!iommu)
1856                 return -ENODEV;
1857
1858         ret = domain_context_mapping_one(domain, iommu, bus, devfn,
1859                                          translation);
1860         if (ret || !dev_is_pci(dev))
1861                 return ret;
1862
1863         /* dependent device mapping */
1864         pdev = to_pci_dev(dev);
1865         tmp = pci_find_upstream_pcie_bridge(pdev);
1866         if (!tmp)
1867                 return 0;
1868         /* Secondary interface's bus number and devfn 0 */
1869         parent = pdev->bus->self;
1870         while (parent != tmp) {
1871                 ret = domain_context_mapping_one(domain, iommu,
1872                                                  parent->bus->number,
1873                                                  parent->devfn, translation);
1874                 if (ret)
1875                         return ret;
1876                 parent = parent->bus->self;
1877         }
1878         if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1879                 return domain_context_mapping_one(domain, iommu,
1880                                         tmp->subordinate->number, 0,
1881                                         translation);
1882         else /* this is a legacy PCI bridge */
1883                 return domain_context_mapping_one(domain, iommu,
1884                                                   tmp->bus->number,
1885                                                   tmp->devfn,
1886                                                   translation);
1887 }
1888
1889 static int domain_context_mapped(struct device *dev)
1890 {
1891         int ret;
1892         struct pci_dev *pdev, *tmp, *parent;
1893         struct intel_iommu *iommu;
1894         u8 bus, devfn;
1895
1896         iommu = device_to_iommu(dev, &bus, &devfn);
1897         if (!iommu)
1898                 return -ENODEV;
1899
1900         ret = device_context_mapped(iommu, bus, devfn);
1901         if (!ret || !dev_is_pci(dev))
1902                 return ret;
1903
1904         /* dependent device mapping */
1905         pdev = to_pci_dev(dev);
1906         tmp = pci_find_upstream_pcie_bridge(pdev);
1907         if (!tmp)
1908                 return ret;
1909         /* Secondary interface's bus number and devfn 0 */
1910         parent = pdev->bus->self;
1911         while (parent != tmp) {
1912                 ret = device_context_mapped(iommu, parent->bus->number,
1913                                             parent->devfn);
1914                 if (!ret)
1915                         return ret;
1916                 parent = parent->bus->self;
1917         }
1918         if (pci_is_pcie(tmp))
1919                 return device_context_mapped(iommu, tmp->subordinate->number,
1920                                              0);
1921         else
1922                 return device_context_mapped(iommu, tmp->bus->number,
1923                                              tmp->devfn);
1924 }
1925
1926 /* Returns a number of VTD pages, but aligned to MM page size */
1927 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1928                                             size_t size)
1929 {
1930         host_addr &= ~PAGE_MASK;
1931         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1932 }
1933
1934 /* Return largest possible superpage level for a given mapping */
1935 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1936                                           unsigned long iov_pfn,
1937                                           unsigned long phy_pfn,
1938                                           unsigned long pages)
1939 {
1940         int support, level = 1;
1941         unsigned long pfnmerge;
1942
1943         support = domain->iommu_superpage;
1944
1945         /* To use a large page, the virtual *and* physical addresses
1946            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1947            of them will mean we have to use smaller pages. So just
1948            merge them and check both at once. */
1949         pfnmerge = iov_pfn | phy_pfn;
1950
1951         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1952                 pages >>= VTD_STRIDE_SHIFT;
1953                 if (!pages)
1954                         break;
1955                 pfnmerge >>= VTD_STRIDE_SHIFT;
1956                 level++;
1957                 support--;
1958         }
1959         return level;
1960 }
1961
1962 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1963                             struct scatterlist *sg, unsigned long phys_pfn,
1964                             unsigned long nr_pages, int prot)
1965 {
1966         struct dma_pte *first_pte = NULL, *pte = NULL;
1967         phys_addr_t uninitialized_var(pteval);
1968         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1969         unsigned long sg_res;
1970         unsigned int largepage_lvl = 0;
1971         unsigned long lvl_pages = 0;
1972
1973         BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1974
1975         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1976                 return -EINVAL;
1977
1978         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1979
1980         if (sg)
1981                 sg_res = 0;
1982         else {
1983                 sg_res = nr_pages + 1;
1984                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1985         }
1986
1987         while (nr_pages > 0) {
1988                 uint64_t tmp;
1989
1990                 if (!sg_res) {
1991                         sg_res = aligned_nrpages(sg->offset, sg->length);
1992                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1993                         sg->dma_length = sg->length;
1994                         pteval = page_to_phys(sg_page(sg)) | prot;
1995                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
1996                 }
1997
1998                 if (!pte) {
1999                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2000
2001                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2002                         if (!pte)
2003                                 return -ENOMEM;
2004                         /* It is large page*/
2005                         if (largepage_lvl > 1) {
2006                                 pteval |= DMA_PTE_LARGE_PAGE;
2007                                 /* Ensure that old small page tables are removed to make room
2008                                    for superpage, if they exist. */
2009                                 dma_pte_clear_range(domain, iov_pfn,
2010                                                     iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2011                                 dma_pte_free_pagetable(domain, iov_pfn,
2012                                                        iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2013                         } else {
2014                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2015                         }
2016
2017                 }
2018                 /* We don't need lock here, nobody else
2019                  * touches the iova range
2020                  */
2021                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2022                 if (tmp) {
2023                         static int dumps = 5;
2024                         printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2025                                iov_pfn, tmp, (unsigned long long)pteval);
2026                         if (dumps) {
2027                                 dumps--;
2028                                 debug_dma_dump_mappings(NULL);
2029                         }
2030                         WARN_ON(1);
2031                 }
2032
2033                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2034
2035                 BUG_ON(nr_pages < lvl_pages);
2036                 BUG_ON(sg_res < lvl_pages);
2037
2038                 nr_pages -= lvl_pages;
2039                 iov_pfn += lvl_pages;
2040                 phys_pfn += lvl_pages;
2041                 pteval += lvl_pages * VTD_PAGE_SIZE;
2042                 sg_res -= lvl_pages;
2043
2044                 /* If the next PTE would be the first in a new page, then we
2045                    need to flush the cache on the entries we've just written.
2046                    And then we'll need to recalculate 'pte', so clear it and
2047                    let it get set again in the if (!pte) block above.
2048
2049                    If we're done (!nr_pages) we need to flush the cache too.
2050
2051                    Also if we've been setting superpages, we may need to
2052                    recalculate 'pte' and switch back to smaller pages for the
2053                    end of the mapping, if the trailing size is not enough to
2054                    use another superpage (i.e. sg_res < lvl_pages). */
2055                 pte++;
2056                 if (!nr_pages || first_pte_in_page(pte) ||
2057                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2058                         domain_flush_cache(domain, first_pte,
2059                                            (void *)pte - (void *)first_pte);
2060                         pte = NULL;
2061                 }
2062
2063                 if (!sg_res && nr_pages)
2064                         sg = sg_next(sg);
2065         }
2066         return 0;
2067 }
2068
2069 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2070                                     struct scatterlist *sg, unsigned long nr_pages,
2071                                     int prot)
2072 {
2073         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2074 }
2075
2076 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2077                                      unsigned long phys_pfn, unsigned long nr_pages,
2078                                      int prot)
2079 {
2080         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2081 }
2082
2083 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2084 {
2085         if (!iommu)
2086                 return;
2087
2088         clear_context_table(iommu, bus, devfn);
2089         iommu->flush.flush_context(iommu, 0, 0, 0,
2090                                            DMA_CCMD_GLOBAL_INVL);
2091         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2092 }
2093
2094 static inline void unlink_domain_info(struct device_domain_info *info)
2095 {
2096         assert_spin_locked(&device_domain_lock);
2097         list_del(&info->link);
2098         list_del(&info->global);
2099         if (info->dev)
2100                 info->dev->archdata.iommu = NULL;
2101 }
2102
2103 static void domain_remove_dev_info(struct dmar_domain *domain)
2104 {
2105         struct device_domain_info *info, *tmp;
2106         unsigned long flags, flags2;
2107
2108         spin_lock_irqsave(&device_domain_lock, flags);
2109         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2110                 unlink_domain_info(info);
2111                 spin_unlock_irqrestore(&device_domain_lock, flags);
2112
2113                 iommu_disable_dev_iotlb(info);
2114                 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2115
2116                 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
2117                         iommu_detach_dependent_devices(info->iommu, info->dev);
2118                         /* clear this iommu in iommu_bmp, update iommu count
2119                          * and capabilities
2120                          */
2121                         spin_lock_irqsave(&domain->iommu_lock, flags2);
2122                         if (test_and_clear_bit(info->iommu->seq_id,
2123                                                domain->iommu_bmp)) {
2124                                 domain->iommu_count--;
2125                                 domain_update_iommu_cap(domain);
2126                         }
2127                         spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2128                 }
2129
2130                 free_devinfo_mem(info);
2131                 spin_lock_irqsave(&device_domain_lock, flags);
2132         }
2133         spin_unlock_irqrestore(&device_domain_lock, flags);
2134 }
2135
2136 /*
2137  * find_domain
2138  * Note: we use struct device->archdata.iommu stores the info
2139  */
2140 static struct dmar_domain *find_domain(struct device *dev)
2141 {
2142         struct device_domain_info *info;
2143
2144         /* No lock here, assumes no domain exit in normal case */
2145         info = dev->archdata.iommu;
2146         if (info)
2147                 return info->domain;
2148         return NULL;
2149 }
2150
2151 static inline struct device_domain_info *
2152 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2153 {
2154         struct device_domain_info *info;
2155
2156         list_for_each_entry(info, &device_domain_list, global)
2157                 if (info->iommu->segment == segment && info->bus == bus &&
2158                     info->devfn == devfn)
2159                         return info;
2160
2161         return NULL;
2162 }
2163
2164 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2165                                                 int bus, int devfn,
2166                                                 struct device *dev,
2167                                                 struct dmar_domain *domain)
2168 {
2169         struct dmar_domain *found = NULL;
2170         struct device_domain_info *info;
2171         unsigned long flags;
2172
2173         info = alloc_devinfo_mem();
2174         if (!info)
2175                 return NULL;
2176
2177         info->bus = bus;
2178         info->devfn = devfn;
2179         info->dev = dev;
2180         info->domain = domain;
2181         info->iommu = iommu;
2182         if (!dev)
2183                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2184
2185         spin_lock_irqsave(&device_domain_lock, flags);
2186         if (dev)
2187                 found = find_domain(dev);
2188         else {
2189                 struct device_domain_info *info2;
2190                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2191                 if (info2)
2192                         found = info2->domain;
2193         }
2194         if (found) {
2195                 spin_unlock_irqrestore(&device_domain_lock, flags);
2196                 free_devinfo_mem(info);
2197                 /* Caller must free the original domain */
2198                 return found;
2199         }
2200
2201         list_add(&info->link, &domain->devices);
2202         list_add(&info->global, &device_domain_list);
2203         if (dev)
2204                 dev->archdata.iommu = info;
2205         spin_unlock_irqrestore(&device_domain_lock, flags);
2206
2207         return domain;
2208 }
2209
2210 /* domain is initialized */
2211 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2212 {
2213         struct dmar_domain *domain, *free = NULL;
2214         struct intel_iommu *iommu = NULL;
2215         struct device_domain_info *info;
2216         struct pci_dev *dev_tmp = NULL;
2217         unsigned long flags;
2218         u8 bus, devfn, bridge_bus, bridge_devfn;
2219
2220         domain = find_domain(dev);
2221         if (domain)
2222                 return domain;
2223
2224         if (dev_is_pci(dev)) {
2225                 struct pci_dev *pdev = to_pci_dev(dev);
2226                 u16 segment;
2227
2228                 segment = pci_domain_nr(pdev->bus);
2229                 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
2230                 if (dev_tmp) {
2231                         if (pci_is_pcie(dev_tmp)) {
2232                                 bridge_bus = dev_tmp->subordinate->number;
2233                                 bridge_devfn = 0;
2234                         } else {
2235                                 bridge_bus = dev_tmp->bus->number;
2236                                 bridge_devfn = dev_tmp->devfn;
2237                         }
2238                         spin_lock_irqsave(&device_domain_lock, flags);
2239                         info = dmar_search_domain_by_dev_info(segment,
2240                                                               bridge_bus,
2241                                                               bridge_devfn);
2242                         if (info) {
2243                                 iommu = info->iommu;
2244                                 domain = info->domain;
2245                         }
2246                         spin_unlock_irqrestore(&device_domain_lock, flags);
2247                         /* pcie-pci bridge already has a domain, uses it */
2248                         if (info)
2249                                 goto found_domain;
2250                 }
2251         }
2252
2253         iommu = device_to_iommu(dev, &bus, &devfn);
2254         if (!iommu)
2255                 goto error;
2256
2257         /* Allocate and initialize new domain for the device */
2258         domain = alloc_domain(false);
2259         if (!domain)
2260                 goto error;
2261         if (iommu_attach_domain(domain, iommu)) {
2262                 free_domain_mem(domain);
2263                 domain = NULL;
2264                 goto error;
2265         }
2266         free = domain;
2267         if (domain_init(domain, gaw))
2268                 goto error;
2269
2270         /* register pcie-to-pci device */
2271         if (dev_tmp) {
2272                 domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn,
2273                                               NULL, domain);
2274                 if (!domain)
2275                         goto error;
2276         }
2277
2278 found_domain:
2279         domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2280 error:
2281         if (free != domain)
2282                 domain_exit(free);
2283
2284         return domain;
2285 }
2286
2287 static int iommu_identity_mapping;
2288 #define IDENTMAP_ALL            1
2289 #define IDENTMAP_GFX            2
2290 #define IDENTMAP_AZALIA         4
2291
2292 static int iommu_domain_identity_map(struct dmar_domain *domain,
2293                                      unsigned long long start,
2294                                      unsigned long long end)
2295 {
2296         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2297         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2298
2299         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2300                           dma_to_mm_pfn(last_vpfn))) {
2301                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2302                 return -ENOMEM;
2303         }
2304
2305         pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2306                  start, end, domain->id);
2307         /*
2308          * RMRR range might have overlap with physical memory range,
2309          * clear it first
2310          */
2311         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2312
2313         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2314                                   last_vpfn - first_vpfn + 1,
2315                                   DMA_PTE_READ|DMA_PTE_WRITE);
2316 }
2317
2318 static int iommu_prepare_identity_map(struct device *dev,
2319                                       unsigned long long start,
2320                                       unsigned long long end)
2321 {
2322         struct dmar_domain *domain;
2323         int ret;
2324
2325         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2326         if (!domain)
2327                 return -ENOMEM;
2328
2329         /* For _hardware_ passthrough, don't bother. But for software
2330            passthrough, we do it anyway -- it may indicate a memory
2331            range which is reserved in E820, so which didn't get set
2332            up to start with in si_domain */
2333         if (domain == si_domain && hw_pass_through) {
2334                 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2335                        dev_name(dev), start, end);
2336                 return 0;
2337         }
2338
2339         printk(KERN_INFO
2340                "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2341                dev_name(dev), start, end);
2342
2343         if (end < start) {
2344                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2345                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2346                         dmi_get_system_info(DMI_BIOS_VENDOR),
2347                         dmi_get_system_info(DMI_BIOS_VERSION),
2348                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2349                 ret = -EIO;
2350                 goto error;
2351         }
2352
2353         if (end >> agaw_to_width(domain->agaw)) {
2354                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2355                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2356                      agaw_to_width(domain->agaw),
2357                      dmi_get_system_info(DMI_BIOS_VENDOR),
2358                      dmi_get_system_info(DMI_BIOS_VERSION),
2359                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2360                 ret = -EIO;
2361                 goto error;
2362         }
2363
2364         ret = iommu_domain_identity_map(domain, start, end);
2365         if (ret)
2366                 goto error;
2367
2368         /* context entry init */
2369         ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2370         if (ret)
2371                 goto error;
2372
2373         return 0;
2374
2375  error:
2376         domain_exit(domain);
2377         return ret;
2378 }
2379
2380 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2381                                          struct device *dev)
2382 {
2383         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2384                 return 0;
2385         return iommu_prepare_identity_map(dev, rmrr->base_address,
2386                                           rmrr->end_address);
2387 }
2388
2389 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2390 static inline void iommu_prepare_isa(void)
2391 {
2392         struct pci_dev *pdev;
2393         int ret;
2394
2395         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2396         if (!pdev)
2397                 return;
2398
2399         printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2400         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2401
2402         if (ret)
2403                 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2404                        "floppy might not work\n");
2405
2406 }
2407 #else
2408 static inline void iommu_prepare_isa(void)
2409 {
2410         return;
2411 }
2412 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2413
2414 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2415
2416 static int __init si_domain_init(int hw)
2417 {
2418         struct dmar_drhd_unit *drhd;
2419         struct intel_iommu *iommu;
2420         int nid, ret = 0;
2421
2422         si_domain = alloc_domain(false);
2423         if (!si_domain)
2424                 return -EFAULT;
2425
2426         si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2427
2428         for_each_active_iommu(iommu, drhd) {
2429                 ret = iommu_attach_domain(si_domain, iommu);
2430                 if (ret) {
2431                         domain_exit(si_domain);
2432                         return -EFAULT;
2433                 }
2434         }
2435
2436         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2437                 domain_exit(si_domain);
2438                 return -EFAULT;
2439         }
2440
2441         pr_debug("IOMMU: identity mapping domain is domain %d\n",
2442                  si_domain->id);
2443
2444         if (hw)
2445                 return 0;
2446
2447         for_each_online_node(nid) {
2448                 unsigned long start_pfn, end_pfn;
2449                 int i;
2450
2451                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2452                         ret = iommu_domain_identity_map(si_domain,
2453                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2454                         if (ret)
2455                                 return ret;
2456                 }
2457         }
2458
2459         return 0;
2460 }
2461
2462 static int identity_mapping(struct device *dev)
2463 {
2464         struct device_domain_info *info;
2465
2466         if (likely(!iommu_identity_mapping))
2467                 return 0;
2468
2469         info = dev->archdata.iommu;
2470         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2471                 return (info->domain == si_domain);
2472
2473         return 0;
2474 }
2475
2476 static int domain_add_dev_info(struct dmar_domain *domain,
2477                                struct device *dev, int translation)
2478 {
2479         struct dmar_domain *ndomain;
2480         struct intel_iommu *iommu;
2481         u8 bus, devfn;
2482         int ret;
2483
2484         iommu = device_to_iommu(dev, &bus, &devfn);
2485         if (!iommu)
2486                 return -ENODEV;
2487
2488         ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2489         if (ndomain != domain)
2490                 return -EBUSY;
2491
2492         ret = domain_context_mapping(domain, dev, translation);
2493         if (ret) {
2494                 domain_remove_one_dev_info(domain, dev);
2495                 return ret;
2496         }
2497
2498         return 0;
2499 }
2500
2501 static bool device_has_rmrr(struct device *dev)
2502 {
2503         struct dmar_rmrr_unit *rmrr;
2504         struct device *tmp;
2505         int i;
2506
2507         rcu_read_lock();
2508         for_each_rmrr_units(rmrr) {
2509                 /*
2510                  * Return TRUE if this RMRR contains the device that
2511                  * is passed in.
2512                  */
2513                 for_each_active_dev_scope(rmrr->devices,
2514                                           rmrr->devices_cnt, i, tmp)
2515                         if (tmp == dev) {
2516                                 rcu_read_unlock();
2517                                 return true;
2518                         }
2519         }
2520         rcu_read_unlock();
2521         return false;
2522 }
2523
2524 static int iommu_should_identity_map(struct device *dev, int startup)
2525 {
2526
2527         if (dev_is_pci(dev)) {
2528                 struct pci_dev *pdev = to_pci_dev(dev);
2529
2530                 /*
2531                  * We want to prevent any device associated with an RMRR from
2532                  * getting placed into the SI Domain. This is done because
2533                  * problems exist when devices are moved in and out of domains
2534                  * and their respective RMRR info is lost. We exempt USB devices
2535                  * from this process due to their usage of RMRRs that are known
2536                  * to not be needed after BIOS hand-off to OS.
2537                  */
2538                 if (device_has_rmrr(dev) &&
2539                     (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2540                         return 0;
2541
2542                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2543                         return 1;
2544
2545                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2546                         return 1;
2547
2548                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2549                         return 0;
2550
2551                 /*
2552                  * We want to start off with all devices in the 1:1 domain, and
2553                  * take them out later if we find they can't access all of memory.
2554                  *
2555                  * However, we can't do this for PCI devices behind bridges,
2556                  * because all PCI devices behind the same bridge will end up
2557                  * with the same source-id on their transactions.
2558                  *
2559                  * Practically speaking, we can't change things around for these
2560                  * devices at run-time, because we can't be sure there'll be no
2561                  * DMA transactions in flight for any of their siblings.
2562                  *
2563                  * So PCI devices (unless they're on the root bus) as well as
2564                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2565                  * the 1:1 domain, just in _case_ one of their siblings turns out
2566                  * not to be able to map all of memory.
2567                  */
2568                 if (!pci_is_pcie(pdev)) {
2569                         if (!pci_is_root_bus(pdev->bus))
2570                                 return 0;
2571                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2572                                 return 0;
2573                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2574                         return 0;
2575         } else {
2576                 if (device_has_rmrr(dev))
2577                         return 0;
2578         }
2579
2580         /*
2581          * At boot time, we don't yet know if devices will be 64-bit capable.
2582          * Assume that they will — if they turn out not to be, then we can
2583          * take them out of the 1:1 domain later.
2584          */
2585         if (!startup) {
2586                 /*
2587                  * If the device's dma_mask is less than the system's memory
2588                  * size then this is not a candidate for identity mapping.
2589                  */
2590                 u64 dma_mask = *dev->dma_mask;
2591
2592                 if (dev->coherent_dma_mask &&
2593                     dev->coherent_dma_mask < dma_mask)
2594                         dma_mask = dev->coherent_dma_mask;
2595
2596                 return dma_mask >= dma_get_required_mask(dev);
2597         }
2598
2599         return 1;
2600 }
2601
2602 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2603 {
2604         int ret;
2605
2606         if (!iommu_should_identity_map(dev, 1))
2607                 return 0;
2608
2609         ret = domain_add_dev_info(si_domain, dev,
2610                                   hw ? CONTEXT_TT_PASS_THROUGH :
2611                                        CONTEXT_TT_MULTI_LEVEL);
2612         if (!ret)
2613                 pr_info("IOMMU: %s identity mapping for device %s\n",
2614                         hw ? "hardware" : "software", dev_name(dev));
2615         else if (ret == -ENODEV)
2616                 /* device not associated with an iommu */
2617                 ret = 0;
2618
2619         return ret;
2620 }
2621
2622
2623 static int __init iommu_prepare_static_identity_mapping(int hw)
2624 {
2625         struct pci_dev *pdev = NULL;
2626         struct dmar_drhd_unit *drhd;
2627         struct intel_iommu *iommu;
2628         struct device *dev;
2629         int i;
2630         int ret = 0;
2631
2632         ret = si_domain_init(hw);
2633         if (ret)
2634                 return -EFAULT;
2635
2636         for_each_pci_dev(pdev) {
2637                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2638                 if (ret)
2639                         return ret;
2640         }
2641
2642         for_each_active_iommu(iommu, drhd)
2643                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2644                         struct acpi_device_physical_node *pn;
2645                         struct acpi_device *adev;
2646
2647                         if (dev->bus != &acpi_bus_type)
2648                                 continue;
2649
2650                         adev= to_acpi_device(dev);
2651                         mutex_lock(&adev->physical_node_lock);
2652                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2653                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2654                                 if (ret)
2655                                         break;
2656                         }
2657                         mutex_unlock(&adev->physical_node_lock);
2658                         if (ret)
2659                                 return ret;
2660                 }
2661
2662         return 0;
2663 }
2664
2665 static int __init init_dmars(void)
2666 {
2667         struct dmar_drhd_unit *drhd;
2668         struct dmar_rmrr_unit *rmrr;
2669         struct device *dev;
2670         struct intel_iommu *iommu;
2671         int i, ret;
2672
2673         /*
2674          * for each drhd
2675          *    allocate root
2676          *    initialize and program root entry to not present
2677          * endfor
2678          */
2679         for_each_drhd_unit(drhd) {
2680                 /*
2681                  * lock not needed as this is only incremented in the single
2682                  * threaded kernel __init code path all other access are read
2683                  * only
2684                  */
2685                 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2686                         g_num_of_iommus++;
2687                         continue;
2688                 }
2689                 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2690                           IOMMU_UNITS_SUPPORTED);
2691         }
2692
2693         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2694                         GFP_KERNEL);
2695         if (!g_iommus) {
2696                 printk(KERN_ERR "Allocating global iommu array failed\n");
2697                 ret = -ENOMEM;
2698                 goto error;
2699         }
2700
2701         deferred_flush = kzalloc(g_num_of_iommus *
2702                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2703         if (!deferred_flush) {
2704                 ret = -ENOMEM;
2705                 goto free_g_iommus;
2706         }
2707
2708         for_each_active_iommu(iommu, drhd) {
2709                 g_iommus[iommu->seq_id] = iommu;
2710
2711                 ret = iommu_init_domains(iommu);
2712                 if (ret)
2713                         goto free_iommu;
2714
2715                 /*
2716                  * TBD:
2717                  * we could share the same root & context tables
2718                  * among all IOMMU's. Need to Split it later.
2719                  */
2720                 ret = iommu_alloc_root_entry(iommu);
2721                 if (ret) {
2722                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2723                         goto free_iommu;
2724                 }
2725                 if (!ecap_pass_through(iommu->ecap))
2726                         hw_pass_through = 0;
2727         }
2728
2729         /*
2730          * Start from the sane iommu hardware state.
2731          */
2732         for_each_active_iommu(iommu, drhd) {
2733                 /*
2734                  * If the queued invalidation is already initialized by us
2735                  * (for example, while enabling interrupt-remapping) then
2736                  * we got the things already rolling from a sane state.
2737                  */
2738                 if (iommu->qi)
2739                         continue;
2740
2741                 /*
2742                  * Clear any previous faults.
2743                  */
2744                 dmar_fault(-1, iommu);
2745                 /*
2746                  * Disable queued invalidation if supported and already enabled
2747                  * before OS handover.
2748                  */
2749                 dmar_disable_qi(iommu);
2750         }
2751
2752         for_each_active_iommu(iommu, drhd) {
2753                 if (dmar_enable_qi(iommu)) {
2754                         /*
2755                          * Queued Invalidate not enabled, use Register Based
2756                          * Invalidate
2757                          */
2758                         iommu->flush.flush_context = __iommu_flush_context;
2759                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2760                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2761                                "invalidation\n",
2762                                 iommu->seq_id,
2763                                (unsigned long long)drhd->reg_base_addr);
2764                 } else {
2765                         iommu->flush.flush_context = qi_flush_context;
2766                         iommu->flush.flush_iotlb = qi_flush_iotlb;
2767                         printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2768                                "invalidation\n",
2769                                 iommu->seq_id,
2770                                (unsigned long long)drhd->reg_base_addr);
2771                 }
2772         }
2773
2774         if (iommu_pass_through)
2775                 iommu_identity_mapping |= IDENTMAP_ALL;
2776
2777 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2778         iommu_identity_mapping |= IDENTMAP_GFX;
2779 #endif
2780
2781         check_tylersburg_isoch();
2782
2783         /*
2784          * If pass through is not set or not enabled, setup context entries for
2785          * identity mappings for rmrr, gfx, and isa and may fall back to static
2786          * identity mapping if iommu_identity_mapping is set.
2787          */
2788         if (iommu_identity_mapping) {
2789                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2790                 if (ret) {
2791                         printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2792                         goto free_iommu;
2793                 }
2794         }
2795         /*
2796          * For each rmrr
2797          *   for each dev attached to rmrr
2798          *   do
2799          *     locate drhd for dev, alloc domain for dev
2800          *     allocate free domain
2801          *     allocate page table entries for rmrr
2802          *     if context not allocated for bus
2803          *           allocate and init context
2804          *           set present in root table for this bus
2805          *     init context with domain, translation etc
2806          *    endfor
2807          * endfor
2808          */
2809         printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2810         for_each_rmrr_units(rmrr) {
2811                 /* some BIOS lists non-exist devices in DMAR table. */
2812                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2813                                           i, dev) {
2814                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
2815                         if (ret)
2816                                 printk(KERN_ERR
2817                                        "IOMMU: mapping reserved region failed\n");
2818                 }
2819         }
2820
2821         iommu_prepare_isa();
2822
2823         /*
2824          * for each drhd
2825          *   enable fault log
2826          *   global invalidate context cache
2827          *   global invalidate iotlb
2828          *   enable translation
2829          */
2830         for_each_iommu(iommu, drhd) {
2831                 if (drhd->ignored) {
2832                         /*
2833                          * we always have to disable PMRs or DMA may fail on
2834                          * this device
2835                          */
2836                         if (force_on)
2837                                 iommu_disable_protect_mem_regions(iommu);
2838                         continue;
2839                 }
2840
2841                 iommu_flush_write_buffer(iommu);
2842
2843                 ret = dmar_set_interrupt(iommu);
2844                 if (ret)
2845                         goto free_iommu;
2846
2847                 iommu_set_root_entry(iommu);
2848
2849                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2850                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2851
2852                 ret = iommu_enable_translation(iommu);
2853                 if (ret)
2854                         goto free_iommu;
2855
2856                 iommu_disable_protect_mem_regions(iommu);
2857         }
2858
2859         return 0;
2860
2861 free_iommu:
2862         for_each_active_iommu(iommu, drhd)
2863                 free_dmar_iommu(iommu);
2864         kfree(deferred_flush);
2865 free_g_iommus:
2866         kfree(g_iommus);
2867 error:
2868         return ret;
2869 }
2870
2871 /* This takes a number of _MM_ pages, not VTD pages */
2872 static struct iova *intel_alloc_iova(struct device *dev,
2873                                      struct dmar_domain *domain,
2874                                      unsigned long nrpages, uint64_t dma_mask)
2875 {
2876         struct iova *iova = NULL;
2877
2878         /* Restrict dma_mask to the width that the iommu can handle */
2879         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2880
2881         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2882                 /*
2883                  * First try to allocate an io virtual address in
2884                  * DMA_BIT_MASK(32) and if that fails then try allocating
2885                  * from higher range
2886                  */
2887                 iova = alloc_iova(&domain->iovad, nrpages,
2888                                   IOVA_PFN(DMA_BIT_MASK(32)), 1);
2889                 if (iova)
2890                         return iova;
2891         }
2892         iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2893         if (unlikely(!iova)) {
2894                 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2895                        nrpages, dev_name(dev));
2896                 return NULL;
2897         }
2898
2899         return iova;
2900 }
2901
2902 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2903 {
2904         struct dmar_domain *domain;
2905         int ret;
2906
2907         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2908         if (!domain) {
2909                 printk(KERN_ERR "Allocating domain for %s failed",
2910                        dev_name(dev));
2911                 return NULL;
2912         }
2913
2914         /* make sure context mapping is ok */
2915         if (unlikely(!domain_context_mapped(dev))) {
2916                 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2917                 if (ret) {
2918                         printk(KERN_ERR "Domain context map for %s failed",
2919                                dev_name(dev));
2920                         return NULL;
2921                 }
2922         }
2923
2924         return domain;
2925 }
2926
2927 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2928 {
2929         struct device_domain_info *info;
2930
2931         /* No lock here, assumes no domain exit in normal case */
2932         info = dev->archdata.iommu;
2933         if (likely(info))
2934                 return info->domain;
2935
2936         return __get_valid_domain_for_dev(dev);
2937 }
2938
2939 static int iommu_dummy(struct device *dev)
2940 {
2941         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2942 }
2943
2944 /* Check if the dev needs to go through non-identity map and unmap process.*/
2945 static int iommu_no_mapping(struct device *dev)
2946 {
2947         int found;
2948
2949         if (iommu_dummy(dev))
2950                 return 1;
2951
2952         if (!iommu_identity_mapping)
2953                 return 0;
2954
2955         found = identity_mapping(dev);
2956         if (found) {
2957                 if (iommu_should_identity_map(dev, 0))
2958                         return 1;
2959                 else {
2960                         /*
2961                          * 32 bit DMA is removed from si_domain and fall back
2962                          * to non-identity mapping.
2963                          */
2964                         domain_remove_one_dev_info(si_domain, dev);
2965                         printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2966                                dev_name(dev));
2967                         return 0;
2968                 }
2969         } else {
2970                 /*
2971                  * In case of a detached 64 bit DMA device from vm, the device
2972                  * is put into si_domain for identity mapping.
2973                  */
2974                 if (iommu_should_identity_map(dev, 0)) {
2975                         int ret;
2976                         ret = domain_add_dev_info(si_domain, dev,
2977                                                   hw_pass_through ?
2978                                                   CONTEXT_TT_PASS_THROUGH :
2979                                                   CONTEXT_TT_MULTI_LEVEL);
2980                         if (!ret) {
2981                                 printk(KERN_INFO "64bit %s uses identity mapping\n",
2982                                        dev_name(dev));
2983                                 return 1;
2984                         }
2985                 }
2986         }
2987
2988         return 0;
2989 }
2990
2991 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
2992                                      size_t size, int dir, u64 dma_mask)
2993 {
2994         struct dmar_domain *domain;
2995         phys_addr_t start_paddr;
2996         struct iova *iova;
2997         int prot = 0;
2998         int ret;
2999         struct intel_iommu *iommu;
3000         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3001
3002         BUG_ON(dir == DMA_NONE);
3003
3004         if (iommu_no_mapping(dev))
3005                 return paddr;
3006
3007         domain = get_valid_domain_for_dev(dev);
3008         if (!domain)
3009                 return 0;
3010
3011         iommu = domain_get_iommu(domain);
3012         size = aligned_nrpages(paddr, size);
3013
3014         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3015         if (!iova)
3016                 goto error;
3017
3018         /*
3019          * Check if DMAR supports zero-length reads on write only
3020          * mappings..
3021          */
3022         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3023                         !cap_zlr(iommu->cap))
3024                 prot |= DMA_PTE_READ;
3025         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3026                 prot |= DMA_PTE_WRITE;
3027         /*
3028          * paddr - (paddr + size) might be partial page, we should map the whole
3029          * page.  Note: if two part of one page are separately mapped, we
3030          * might have two guest_addr mapping to the same host paddr, but this
3031          * is not a big problem
3032          */
3033         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3034                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3035         if (ret)
3036                 goto error;
3037
3038         /* it's a non-present to present mapping. Only flush if caching mode */
3039         if (cap_caching_mode(iommu->cap))
3040                 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3041         else
3042                 iommu_flush_write_buffer(iommu);
3043
3044         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3045         start_paddr += paddr & ~PAGE_MASK;
3046         return start_paddr;
3047
3048 error:
3049         if (iova)
3050                 __free_iova(&domain->iovad, iova);
3051         printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
3052                 dev_name(dev), size, (unsigned long long)paddr, dir);
3053         return 0;
3054 }
3055
3056 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3057                                  unsigned long offset, size_t size,
3058                                  enum dma_data_direction dir,
3059                                  struct dma_attrs *attrs)
3060 {
3061         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3062                                   dir, *dev->dma_mask);
3063 }
3064
3065 static void flush_unmaps(void)
3066 {
3067         int i, j;
3068
3069         timer_on = 0;
3070
3071         /* just flush them all */
3072         for (i = 0; i < g_num_of_iommus; i++) {
3073                 struct intel_iommu *iommu = g_iommus[i];
3074                 if (!iommu)
3075                         continue;
3076
3077                 if (!deferred_flush[i].next)
3078                         continue;
3079
3080                 /* In caching mode, global flushes turn emulation expensive */
3081                 if (!cap_caching_mode(iommu->cap))
3082                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3083                                          DMA_TLB_GLOBAL_FLUSH);
3084                 for (j = 0; j < deferred_flush[i].next; j++) {
3085                         unsigned long mask;
3086                         struct iova *iova = deferred_flush[i].iova[j];
3087                         struct dmar_domain *domain = deferred_flush[i].domain[j];
3088
3089                         /* On real hardware multiple invalidations are expensive */
3090                         if (cap_caching_mode(iommu->cap))
3091                                 iommu_flush_iotlb_psi(iommu, domain->id,
3092                                         iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3093                                         !deferred_flush[i].freelist[j], 0);
3094                         else {
3095                                 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3096                                 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3097                                                 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3098                         }
3099                         __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3100                         if (deferred_flush[i].freelist[j])
3101                                 dma_free_pagelist(deferred_flush[i].freelist[j]);
3102                 }
3103                 deferred_flush[i].next = 0;
3104         }
3105
3106         list_size = 0;
3107 }
3108
3109 static void flush_unmaps_timeout(unsigned long data)
3110 {
3111         unsigned long flags;
3112
3113         spin_lock_irqsave(&async_umap_flush_lock, flags);
3114         flush_unmaps();
3115         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3116 }
3117
3118 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3119 {
3120         unsigned long flags;
3121         int next, iommu_id;
3122         struct intel_iommu *iommu;
3123
3124         spin_lock_irqsave(&async_umap_flush_lock, flags);
3125         if (list_size == HIGH_WATER_MARK)
3126                 flush_unmaps();
3127
3128         iommu = domain_get_iommu(dom);
3129         iommu_id = iommu->seq_id;
3130
3131         next = deferred_flush[iommu_id].next;
3132         deferred_flush[iommu_id].domain[next] = dom;
3133         deferred_flush[iommu_id].iova[next] = iova;
3134         deferred_flush[iommu_id].freelist[next] = freelist;
3135         deferred_flush[iommu_id].next++;
3136
3137         if (!timer_on) {
3138                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3139                 timer_on = 1;
3140         }
3141         list_size++;
3142         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3143 }
3144
3145 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3146                              size_t size, enum dma_data_direction dir,
3147                              struct dma_attrs *attrs)
3148 {
3149         struct dmar_domain *domain;
3150         unsigned long start_pfn, last_pfn;
3151         struct iova *iova;
3152         struct intel_iommu *iommu;
3153         struct page *freelist;
3154
3155         if (iommu_no_mapping(dev))
3156                 return;
3157
3158         domain = find_domain(dev);
3159         BUG_ON(!domain);
3160
3161         iommu = domain_get_iommu(domain);
3162
3163         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3164         if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3165                       (unsigned long long)dev_addr))
3166                 return;
3167
3168         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3169         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3170
3171         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3172                  dev_name(dev), start_pfn, last_pfn);
3173
3174         freelist = domain_unmap(domain, start_pfn, last_pfn);
3175
3176         if (intel_iommu_strict) {
3177                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3178                                       last_pfn - start_pfn + 1, !freelist, 0);
3179                 /* free iova */
3180                 __free_iova(&domain->iovad, iova);
3181                 dma_free_pagelist(freelist);
3182         } else {
3183                 add_unmap(domain, iova, freelist);
3184                 /*
3185                  * queue up the release of the unmap to save the 1/6th of the
3186                  * cpu used up by the iotlb flush operation...
3187                  */
3188         }
3189 }
3190
3191 static void *intel_alloc_coherent(struct device *dev, size_t size,
3192                                   dma_addr_t *dma_handle, gfp_t flags,
3193                                   struct dma_attrs *attrs)
3194 {
3195         struct page *page = NULL;
3196         int order;
3197
3198         size = PAGE_ALIGN(size);
3199         order = get_order(size);
3200
3201         if (!iommu_no_mapping(dev))
3202                 flags &= ~(GFP_DMA | GFP_DMA32);
3203         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3204                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3205                         flags |= GFP_DMA;
3206                 else
3207                         flags |= GFP_DMA32;
3208         }
3209
3210         if (flags & __GFP_WAIT) {
3211                 unsigned int count = size >> PAGE_SHIFT;
3212
3213                 page = dma_alloc_from_contiguous(dev, count, order);
3214                 if (page && iommu_no_mapping(dev) &&
3215                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3216                         dma_release_from_contiguous(dev, page, count);
3217                         page = NULL;
3218                 }
3219         }
3220
3221         if (!page)
3222                 page = alloc_pages(flags, order);
3223         if (!page)
3224                 return NULL;
3225         memset(page_address(page), 0, size);
3226
3227         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3228                                          DMA_BIDIRECTIONAL,
3229                                          dev->coherent_dma_mask);
3230         if (*dma_handle)
3231                 return page_address(page);
3232         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3233                 __free_pages(page, order);
3234
3235         return NULL;
3236 }
3237
3238 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3239                                 dma_addr_t dma_handle, struct dma_attrs *attrs)
3240 {
3241         int order;
3242         struct page *page = virt_to_page(vaddr);
3243
3244         size = PAGE_ALIGN(size);
3245         order = get_order(size);
3246
3247         intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
3248         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3249                 __free_pages(page, order);
3250 }
3251
3252 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3253                            int nelems, enum dma_data_direction dir,
3254                            struct dma_attrs *attrs)
3255 {
3256         struct dmar_domain *domain;
3257         unsigned long start_pfn, last_pfn;
3258         struct iova *iova;
3259         struct intel_iommu *iommu;
3260         struct page *freelist;
3261
3262         if (iommu_no_mapping(dev))
3263                 return;
3264
3265         domain = find_domain(dev);
3266         BUG_ON(!domain);
3267
3268         iommu = domain_get_iommu(domain);
3269
3270         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3271         if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3272                       (unsigned long long)sglist[0].dma_address))
3273                 return;
3274
3275         start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3276         last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3277
3278         freelist = domain_unmap(domain, start_pfn, last_pfn);
3279
3280         if (intel_iommu_strict) {
3281                 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3282                                       last_pfn - start_pfn + 1, !freelist, 0);
3283                 /* free iova */
3284                 __free_iova(&domain->iovad, iova);
3285                 dma_free_pagelist(freelist);
3286         } else {
3287                 add_unmap(domain, iova, freelist);
3288                 /*
3289                  * queue up the release of the unmap to save the 1/6th of the
3290                  * cpu used up by the iotlb flush operation...
3291                  */
3292         }
3293 }
3294
3295 static int intel_nontranslate_map_sg(struct device *hddev,
3296         struct scatterlist *sglist, int nelems, int dir)
3297 {
3298         int i;
3299         struct scatterlist *sg;
3300
3301         for_each_sg(sglist, sg, nelems, i) {
3302                 BUG_ON(!sg_page(sg));
3303                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3304                 sg->dma_length = sg->length;
3305         }
3306         return nelems;
3307 }
3308
3309 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3310                         enum dma_data_direction dir, struct dma_attrs *attrs)
3311 {
3312         int i;
3313         struct dmar_domain *domain;
3314         size_t size = 0;
3315         int prot = 0;
3316         struct iova *iova = NULL;
3317         int ret;
3318         struct scatterlist *sg;
3319         unsigned long start_vpfn;
3320         struct intel_iommu *iommu;
3321
3322         BUG_ON(dir == DMA_NONE);
3323         if (iommu_no_mapping(dev))
3324                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3325
3326         domain = get_valid_domain_for_dev(dev);
3327         if (!domain)
3328                 return 0;
3329
3330         iommu = domain_get_iommu(domain);
3331
3332         for_each_sg(sglist, sg, nelems, i)
3333                 size += aligned_nrpages(sg->offset, sg->length);
3334
3335         iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3336                                 *dev->dma_mask);
3337         if (!iova) {
3338                 sglist->dma_length = 0;
3339                 return 0;
3340         }
3341
3342         /*
3343          * Check if DMAR supports zero-length reads on write only
3344          * mappings..
3345          */
3346         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3347                         !cap_zlr(iommu->cap))
3348                 prot |= DMA_PTE_READ;
3349         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3350                 prot |= DMA_PTE_WRITE;
3351
3352         start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3353
3354         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3355         if (unlikely(ret)) {
3356                 /*  clear the page */
3357                 dma_pte_clear_range(domain, start_vpfn,
3358                                     start_vpfn + size - 1);
3359                 /* free page tables */
3360                 dma_pte_free_pagetable(domain, start_vpfn,
3361                                        start_vpfn + size - 1);
3362                 /* free iova */
3363                 __free_iova(&domain->iovad, iova);
3364                 return 0;
3365         }
3366
3367         /* it's a non-present to present mapping. Only flush if caching mode */
3368         if (cap_caching_mode(iommu->cap))
3369                 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3370         else
3371                 iommu_flush_write_buffer(iommu);
3372
3373         return nelems;
3374 }
3375
3376 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3377 {
3378         return !dma_addr;
3379 }
3380
3381 struct dma_map_ops intel_dma_ops = {
3382         .alloc = intel_alloc_coherent,
3383         .free = intel_free_coherent,
3384         .map_sg = intel_map_sg,
3385         .unmap_sg = intel_unmap_sg,
3386         .map_page = intel_map_page,
3387         .unmap_page = intel_unmap_page,
3388         .mapping_error = intel_mapping_error,
3389 };
3390
3391 static inline int iommu_domain_cache_init(void)
3392 {
3393         int ret = 0;
3394
3395         iommu_domain_cache = kmem_cache_create("iommu_domain",
3396                                          sizeof(struct dmar_domain),
3397                                          0,
3398                                          SLAB_HWCACHE_ALIGN,
3399
3400                                          NULL);
3401         if (!iommu_domain_cache) {
3402                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3403                 ret = -ENOMEM;
3404         }
3405
3406         return ret;
3407 }
3408
3409 static inline int iommu_devinfo_cache_init(void)
3410 {
3411         int ret = 0;
3412
3413         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3414                                          sizeof(struct device_domain_info),
3415                                          0,
3416                                          SLAB_HWCACHE_ALIGN,
3417                                          NULL);
3418         if (!iommu_devinfo_cache) {
3419                 printk(KERN_ERR "Couldn't create devinfo cache\n");
3420                 ret = -ENOMEM;
3421         }
3422
3423         return ret;
3424 }
3425
3426 static inline int iommu_iova_cache_init(void)
3427 {
3428         int ret = 0;
3429
3430         iommu_iova_cache = kmem_cache_create("iommu_iova",
3431                                          sizeof(struct iova),
3432                                          0,
3433                                          SLAB_HWCACHE_ALIGN,
3434                                          NULL);
3435         if (!iommu_iova_cache) {
3436                 printk(KERN_ERR "Couldn't create iova cache\n");
3437                 ret = -ENOMEM;
3438         }
3439
3440         return ret;
3441 }
3442
3443 static int __init iommu_init_mempool(void)
3444 {
3445         int ret;
3446         ret = iommu_iova_cache_init();
3447         if (ret)
3448                 return ret;
3449
3450         ret = iommu_domain_cache_init();
3451         if (ret)
3452                 goto domain_error;
3453
3454         ret = iommu_devinfo_cache_init();
3455         if (!ret)
3456                 return ret;
3457
3458         kmem_cache_destroy(iommu_domain_cache);
3459 domain_error:
3460         kmem_cache_destroy(iommu_iova_cache);
3461
3462         return -ENOMEM;
3463 }
3464
3465 static void __init iommu_exit_mempool(void)
3466 {
3467         kmem_cache_destroy(iommu_devinfo_cache);
3468         kmem_cache_destroy(iommu_domain_cache);
3469         kmem_cache_destroy(iommu_iova_cache);
3470
3471 }
3472
3473 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3474 {
3475         struct dmar_drhd_unit *drhd;
3476         u32 vtbar;
3477         int rc;
3478
3479         /* We know that this device on this chipset has its own IOMMU.
3480          * If we find it under a different IOMMU, then the BIOS is lying
3481          * to us. Hope that the IOMMU for this device is actually
3482          * disabled, and it needs no translation...
3483          */
3484         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3485         if (rc) {
3486                 /* "can't" happen */
3487                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3488                 return;
3489         }
3490         vtbar &= 0xffff0000;
3491
3492         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3493         drhd = dmar_find_matched_drhd_unit(pdev);
3494         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3495                             TAINT_FIRMWARE_WORKAROUND,
3496                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3497                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3498 }
3499 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3500
3501 static void __init init_no_remapping_devices(void)
3502 {
3503         struct dmar_drhd_unit *drhd;
3504         struct device *dev;
3505         int i;
3506
3507         for_each_drhd_unit(drhd) {
3508                 if (!drhd->include_all) {
3509                         for_each_active_dev_scope(drhd->devices,
3510                                                   drhd->devices_cnt, i, dev)
3511                                 break;
3512                         /* ignore DMAR unit if no devices exist */
3513                         if (i == drhd->devices_cnt)
3514                                 drhd->ignored = 1;
3515                 }
3516         }
3517
3518         for_each_active_drhd_unit(drhd) {
3519                 if (drhd->include_all)
3520                         continue;
3521
3522                 for_each_active_dev_scope(drhd->devices,
3523                                           drhd->devices_cnt, i, dev)
3524                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3525                                 break;
3526                 if (i < drhd->devices_cnt)
3527                         continue;
3528
3529                 /* This IOMMU has *only* gfx devices. Either bypass it or
3530                    set the gfx_mapped flag, as appropriate */
3531                 if (dmar_map_gfx) {
3532                         intel_iommu_gfx_mapped = 1;
3533                 } else {
3534                         drhd->ignored = 1;
3535                         for_each_active_dev_scope(drhd->devices,
3536                                                   drhd->devices_cnt, i, dev)
3537                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3538                 }
3539         }
3540 }
3541
3542 #ifdef CONFIG_SUSPEND
3543 static int init_iommu_hw(void)
3544 {
3545         struct dmar_drhd_unit *drhd;
3546         struct intel_iommu *iommu = NULL;
3547
3548         for_each_active_iommu(iommu, drhd)
3549                 if (iommu->qi)
3550                         dmar_reenable_qi(iommu);
3551
3552         for_each_iommu(iommu, drhd) {
3553                 if (drhd->ignored) {
3554                         /*
3555                          * we always have to disable PMRs or DMA may fail on
3556                          * this device
3557                          */
3558                         if (force_on)
3559                                 iommu_disable_protect_mem_regions(iommu);
3560                         continue;
3561                 }
3562
3563                 iommu_flush_write_buffer(iommu);
3564
3565                 iommu_set_root_entry(iommu);
3566
3567                 iommu->flush.flush_context(iommu, 0, 0, 0,
3568                                            DMA_CCMD_GLOBAL_INVL);
3569                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3570                                          DMA_TLB_GLOBAL_FLUSH);
3571                 if (iommu_enable_translation(iommu))
3572                         return 1;
3573                 iommu_disable_protect_mem_regions(iommu);
3574         }
3575
3576         return 0;
3577 }
3578
3579 static void iommu_flush_all(void)
3580 {
3581         struct dmar_drhd_unit *drhd;
3582         struct intel_iommu *iommu;
3583
3584         for_each_active_iommu(iommu, drhd) {
3585                 iommu->flush.flush_context(iommu, 0, 0, 0,
3586                                            DMA_CCMD_GLOBAL_INVL);
3587                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3588                                          DMA_TLB_GLOBAL_FLUSH);
3589         }
3590 }
3591
3592 static int iommu_suspend(void)
3593 {
3594         struct dmar_drhd_unit *drhd;
3595         struct intel_iommu *iommu = NULL;
3596         unsigned long flag;
3597
3598         for_each_active_iommu(iommu, drhd) {
3599                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3600                                                  GFP_ATOMIC);
3601                 if (!iommu->iommu_state)
3602                         goto nomem;
3603         }
3604
3605         iommu_flush_all();
3606
3607         for_each_active_iommu(iommu, drhd) {
3608                 iommu_disable_translation(iommu);
3609
3610                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3611
3612                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3613                         readl(iommu->reg + DMAR_FECTL_REG);
3614                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3615                         readl(iommu->reg + DMAR_FEDATA_REG);
3616                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3617                         readl(iommu->reg + DMAR_FEADDR_REG);
3618                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3619                         readl(iommu->reg + DMAR_FEUADDR_REG);
3620
3621                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3622         }
3623         return 0;
3624
3625 nomem:
3626         for_each_active_iommu(iommu, drhd)
3627                 kfree(iommu->iommu_state);
3628
3629         return -ENOMEM;
3630 }
3631
3632 static void iommu_resume(void)
3633 {
3634         struct dmar_drhd_unit *drhd;
3635         struct intel_iommu *iommu = NULL;
3636         unsigned long flag;
3637
3638         if (init_iommu_hw()) {
3639                 if (force_on)
3640                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3641                 else
3642                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3643                 return;
3644         }
3645
3646         for_each_active_iommu(iommu, drhd) {
3647
3648                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3649
3650                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3651                         iommu->reg + DMAR_FECTL_REG);
3652                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3653                         iommu->reg + DMAR_FEDATA_REG);
3654                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3655                         iommu->reg + DMAR_FEADDR_REG);
3656                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3657                         iommu->reg + DMAR_FEUADDR_REG);
3658
3659                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3660         }
3661
3662         for_each_active_iommu(iommu, drhd)
3663                 kfree(iommu->iommu_state);
3664 }
3665
3666 static struct syscore_ops iommu_syscore_ops = {
3667         .resume         = iommu_resume,
3668         .suspend        = iommu_suspend,
3669 };
3670
3671 static void __init init_iommu_pm_ops(void)
3672 {
3673         register_syscore_ops(&iommu_syscore_ops);
3674 }
3675
3676 #else
3677 static inline void init_iommu_pm_ops(void) {}
3678 #endif  /* CONFIG_PM */
3679
3680
3681 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3682 {
3683         struct acpi_dmar_reserved_memory *rmrr;
3684         struct dmar_rmrr_unit *rmrru;
3685
3686         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3687         if (!rmrru)
3688                 return -ENOMEM;
3689
3690         rmrru->hdr = header;
3691         rmrr = (struct acpi_dmar_reserved_memory *)header;
3692         rmrru->base_address = rmrr->base_address;
3693         rmrru->end_address = rmrr->end_address;
3694         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3695                                 ((void *)rmrr) + rmrr->header.length,
3696                                 &rmrru->devices_cnt);
3697         if (rmrru->devices_cnt && rmrru->devices == NULL) {
3698                 kfree(rmrru);
3699                 return -ENOMEM;
3700         }
3701
3702         list_add(&rmrru->list, &dmar_rmrr_units);
3703
3704         return 0;
3705 }
3706
3707 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3708 {
3709         struct acpi_dmar_atsr *atsr;
3710         struct dmar_atsr_unit *atsru;
3711
3712         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3713         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3714         if (!atsru)
3715                 return -ENOMEM;
3716
3717         atsru->hdr = hdr;
3718         atsru->include_all = atsr->flags & 0x1;
3719         if (!atsru->include_all) {
3720                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3721                                 (void *)atsr + atsr->header.length,
3722                                 &atsru->devices_cnt);
3723                 if (atsru->devices_cnt && atsru->devices == NULL) {
3724                         kfree(atsru);
3725                         return -ENOMEM;
3726                 }
3727         }
3728
3729         list_add_rcu(&atsru->list, &dmar_atsr_units);
3730
3731         return 0;
3732 }
3733
3734 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3735 {
3736         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3737         kfree(atsru);
3738 }
3739
3740 static void intel_iommu_free_dmars(void)
3741 {
3742         struct dmar_rmrr_unit *rmrru, *rmrr_n;
3743         struct dmar_atsr_unit *atsru, *atsr_n;
3744
3745         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3746                 list_del(&rmrru->list);
3747                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3748                 kfree(rmrru);
3749         }
3750
3751         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3752                 list_del(&atsru->list);
3753                 intel_iommu_free_atsr(atsru);
3754         }
3755 }
3756
3757 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3758 {
3759         int i, ret = 1;
3760         struct pci_bus *bus;
3761         struct pci_dev *bridge = NULL;
3762         struct device *tmp;
3763         struct acpi_dmar_atsr *atsr;
3764         struct dmar_atsr_unit *atsru;
3765
3766         dev = pci_physfn(dev);
3767         for (bus = dev->bus; bus; bus = bus->parent) {
3768                 bridge = bus->self;
3769                 if (!bridge || !pci_is_pcie(bridge) ||
3770                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3771                         return 0;
3772                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3773                         break;
3774         }
3775         if (!bridge)
3776                 return 0;
3777
3778         rcu_read_lock();
3779         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3780                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3781                 if (atsr->segment != pci_domain_nr(dev->bus))
3782                         continue;
3783
3784                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3785                         if (tmp == &bridge->dev)
3786                                 goto out;
3787
3788                 if (atsru->include_all)
3789                         goto out;
3790         }
3791         ret = 0;
3792 out:
3793         rcu_read_unlock();
3794
3795         return ret;
3796 }
3797
3798 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3799 {
3800         int ret = 0;
3801         struct dmar_rmrr_unit *rmrru;
3802         struct dmar_atsr_unit *atsru;
3803         struct acpi_dmar_atsr *atsr;
3804         struct acpi_dmar_reserved_memory *rmrr;
3805
3806         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3807                 return 0;
3808
3809         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3810                 rmrr = container_of(rmrru->hdr,
3811                                     struct acpi_dmar_reserved_memory, header);
3812                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3813                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3814                                 ((void *)rmrr) + rmrr->header.length,
3815                                 rmrr->segment, rmrru->devices,
3816                                 rmrru->devices_cnt);
3817                         if(ret < 0)
3818                                 return ret;
3819                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3820                         dmar_remove_dev_scope(info, rmrr->segment,
3821                                 rmrru->devices, rmrru->devices_cnt);
3822                 }
3823         }
3824
3825         list_for_each_entry(atsru, &dmar_atsr_units, list) {
3826                 if (atsru->include_all)
3827                         continue;
3828
3829                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3830                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3831                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3832                                         (void *)atsr + atsr->header.length,
3833                                         atsr->segment, atsru->devices,
3834                                         atsru->devices_cnt);
3835                         if (ret > 0)
3836                                 break;
3837                         else if(ret < 0)
3838                                 return ret;
3839                 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3840                         if (dmar_remove_dev_scope(info, atsr->segment,
3841                                         atsru->devices, atsru->devices_cnt))
3842                                 break;
3843                 }
3844         }
3845
3846         return 0;
3847 }
3848
3849 /*
3850  * Here we only respond to action of unbound device from driver.
3851  *
3852  * Added device is not attached to its DMAR domain here yet. That will happen
3853  * when mapping the device to iova.
3854  */
3855 static int device_notifier(struct notifier_block *nb,
3856                                   unsigned long action, void *data)
3857 {
3858         struct device *dev = data;
3859         struct dmar_domain *domain;
3860
3861         if (iommu_dummy(dev))
3862                 return 0;
3863
3864         if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3865             action != BUS_NOTIFY_DEL_DEVICE)
3866                 return 0;
3867
3868         domain = find_domain(dev);
3869         if (!domain)
3870                 return 0;
3871
3872         down_read(&dmar_global_lock);
3873         domain_remove_one_dev_info(domain, dev);
3874         if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3875             !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3876             list_empty(&domain->devices))
3877                 domain_exit(domain);
3878         up_read(&dmar_global_lock);
3879
3880         return 0;
3881 }
3882
3883 static struct notifier_block device_nb = {
3884         .notifier_call = device_notifier,
3885 };
3886
3887 static int intel_iommu_memory_notifier(struct notifier_block *nb,
3888                                        unsigned long val, void *v)
3889 {
3890         struct memory_notify *mhp = v;
3891         unsigned long long start, end;
3892         unsigned long start_vpfn, last_vpfn;
3893
3894         switch (val) {
3895         case MEM_GOING_ONLINE:
3896                 start = mhp->start_pfn << PAGE_SHIFT;
3897                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3898                 if (iommu_domain_identity_map(si_domain, start, end)) {
3899                         pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3900                                 start, end);
3901                         return NOTIFY_BAD;
3902                 }
3903                 break;
3904
3905         case MEM_OFFLINE:
3906         case MEM_CANCEL_ONLINE:
3907                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3908                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3909                 while (start_vpfn <= last_vpfn) {
3910                         struct iova *iova;
3911                         struct dmar_drhd_unit *drhd;
3912                         struct intel_iommu *iommu;
3913                         struct page *freelist;
3914
3915                         iova = find_iova(&si_domain->iovad, start_vpfn);
3916                         if (iova == NULL) {
3917                                 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3918                                          start_vpfn);
3919                                 break;
3920                         }
3921
3922                         iova = split_and_remove_iova(&si_domain->iovad, iova,
3923                                                      start_vpfn, last_vpfn);
3924                         if (iova == NULL) {
3925                                 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3926                                         start_vpfn, last_vpfn);
3927                                 return NOTIFY_BAD;
3928                         }
3929
3930                         freelist = domain_unmap(si_domain, iova->pfn_lo,
3931                                                iova->pfn_hi);
3932
3933                         rcu_read_lock();
3934                         for_each_active_iommu(iommu, drhd)
3935                                 iommu_flush_iotlb_psi(iommu, si_domain->id,
3936                                         iova->pfn_lo,
3937                                         iova->pfn_hi - iova->pfn_lo + 1,
3938                                         !freelist, 0);
3939                         rcu_read_unlock();
3940                         dma_free_pagelist(freelist);
3941
3942                         start_vpfn = iova->pfn_hi + 1;
3943                         free_iova_mem(iova);
3944                 }
3945                 break;
3946         }
3947
3948         return NOTIFY_OK;
3949 }
3950
3951 static struct notifier_block intel_iommu_memory_nb = {
3952         .notifier_call = intel_iommu_memory_notifier,
3953         .priority = 0
3954 };
3955
3956 int __init intel_iommu_init(void)
3957 {
3958         int ret = -ENODEV;
3959         struct dmar_drhd_unit *drhd;
3960         struct intel_iommu *iommu;
3961
3962         /* VT-d is required for a TXT/tboot launch, so enforce that */
3963         force_on = tboot_force_iommu();
3964
3965         if (iommu_init_mempool()) {
3966                 if (force_on)
3967                         panic("tboot: Failed to initialize iommu memory\n");
3968                 return -ENOMEM;
3969         }
3970
3971         down_write(&dmar_global_lock);
3972         if (dmar_table_init()) {
3973                 if (force_on)
3974                         panic("tboot: Failed to initialize DMAR table\n");
3975                 goto out_free_dmar;
3976         }
3977
3978         /*
3979          * Disable translation if already enabled prior to OS handover.
3980          */
3981         for_each_active_iommu(iommu, drhd)
3982                 if (iommu->gcmd & DMA_GCMD_TE)
3983                         iommu_disable_translation(iommu);
3984
3985         if (dmar_dev_scope_init() < 0) {
3986                 if (force_on)
3987                         panic("tboot: Failed to initialize DMAR device scope\n");
3988                 goto out_free_dmar;
3989         }
3990
3991         if (no_iommu || dmar_disabled)
3992                 goto out_free_dmar;
3993
3994         if (list_empty(&dmar_rmrr_units))
3995                 printk(KERN_INFO "DMAR: No RMRR found\n");
3996
3997         if (list_empty(&dmar_atsr_units))
3998                 printk(KERN_INFO "DMAR: No ATSR found\n");
3999
4000         if (dmar_init_reserved_ranges()) {
4001                 if (force_on)
4002                         panic("tboot: Failed to reserve iommu ranges\n");
4003                 goto out_free_reserved_range;
4004         }
4005
4006         init_no_remapping_devices();
4007
4008         ret = init_dmars();
4009         if (ret) {
4010                 if (force_on)
4011                         panic("tboot: Failed to initialize DMARs\n");
4012                 printk(KERN_ERR "IOMMU: dmar init failed\n");
4013                 goto out_free_reserved_range;
4014         }
4015         up_write(&dmar_global_lock);
4016         printk(KERN_INFO
4017         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4018
4019         init_timer(&unmap_timer);
4020 #ifdef CONFIG_SWIOTLB
4021         swiotlb = 0;
4022 #endif
4023         dma_ops = &intel_dma_ops;
4024
4025         init_iommu_pm_ops();
4026
4027         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4028         bus_register_notifier(&pci_bus_type, &device_nb);
4029         if (si_domain && !hw_pass_through)
4030                 register_memory_notifier(&intel_iommu_memory_nb);
4031
4032         intel_iommu_enabled = 1;
4033
4034         return 0;
4035
4036 out_free_reserved_range:
4037         put_iova_domain(&reserved_iova_list);
4038 out_free_dmar:
4039         intel_iommu_free_dmars();
4040         up_write(&dmar_global_lock);
4041         iommu_exit_mempool();
4042         return ret;
4043 }
4044
4045 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4046                                            struct device *dev)
4047 {
4048         struct pci_dev *tmp, *parent, *pdev;
4049
4050         if (!iommu || !dev || !dev_is_pci(dev))
4051                 return;
4052
4053         pdev = to_pci_dev(dev);
4054
4055         /* dependent device detach */
4056         tmp = pci_find_upstream_pcie_bridge(pdev);
4057         /* Secondary interface's bus number and devfn 0 */
4058         if (tmp) {
4059                 parent = pdev->bus->self;
4060                 while (parent != tmp) {
4061                         iommu_detach_dev(iommu, parent->bus->number,
4062                                          parent->devfn);
4063                         parent = parent->bus->self;
4064                 }
4065                 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
4066                         iommu_detach_dev(iommu,
4067                                 tmp->subordinate->number, 0);
4068                 else /* this is a legacy PCI bridge */
4069                         iommu_detach_dev(iommu, tmp->bus->number,
4070                                          tmp->devfn);
4071         }
4072 }
4073
4074 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4075                                        struct device *dev)
4076 {
4077         struct device_domain_info *info, *tmp;
4078         struct intel_iommu *iommu;
4079         unsigned long flags;
4080         int found = 0;
4081         u8 bus, devfn;
4082
4083         iommu = device_to_iommu(dev, &bus, &devfn);
4084         if (!iommu)
4085                 return;
4086
4087         spin_lock_irqsave(&device_domain_lock, flags);
4088         list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4089                 if (info->iommu == iommu && info->bus == bus &&
4090                     info->devfn == devfn) {
4091                         unlink_domain_info(info);
4092                         spin_unlock_irqrestore(&device_domain_lock, flags);
4093
4094                         iommu_disable_dev_iotlb(info);
4095                         iommu_detach_dev(iommu, info->bus, info->devfn);
4096                         iommu_detach_dependent_devices(iommu, dev);
4097                         free_devinfo_mem(info);
4098
4099                         spin_lock_irqsave(&device_domain_lock, flags);
4100
4101                         if (found)
4102                                 break;
4103                         else
4104                                 continue;
4105                 }
4106
4107                 /* if there is no other devices under the same iommu
4108                  * owned by this domain, clear this iommu in iommu_bmp
4109                  * update iommu count and coherency
4110                  */
4111                 if (info->iommu == iommu)
4112                         found = 1;
4113         }
4114
4115         spin_unlock_irqrestore(&device_domain_lock, flags);
4116
4117         if (found == 0) {
4118                 unsigned long tmp_flags;
4119                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
4120                 clear_bit(iommu->seq_id, domain->iommu_bmp);
4121                 domain->iommu_count--;
4122                 domain_update_iommu_cap(domain);
4123                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
4124
4125                 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4126                     !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4127                         spin_lock_irqsave(&iommu->lock, tmp_flags);
4128                         clear_bit(domain->id, iommu->domain_ids);
4129                         iommu->domains[domain->id] = NULL;
4130                         spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4131                 }
4132         }
4133 }
4134
4135 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4136 {
4137         int adjust_width;
4138
4139         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
4140         domain_reserve_special_ranges(domain);
4141
4142         /* calculate AGAW */
4143         domain->gaw = guest_width;
4144         adjust_width = guestwidth_to_adjustwidth(guest_width);
4145         domain->agaw = width_to_agaw(adjust_width);
4146
4147         domain->iommu_coherency = 0;
4148         domain->iommu_snooping = 0;
4149         domain->iommu_superpage = 0;
4150         domain->max_addr = 0;
4151
4152         /* always allocate the top pgd */
4153         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4154         if (!domain->pgd)
4155                 return -ENOMEM;
4156         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4157         return 0;
4158 }
4159
4160 static int intel_iommu_domain_init(struct iommu_domain *domain)
4161 {
4162         struct dmar_domain *dmar_domain;
4163
4164         dmar_domain = alloc_domain(true);
4165         if (!dmar_domain) {
4166                 printk(KERN_ERR
4167                         "intel_iommu_domain_init: dmar_domain == NULL\n");
4168                 return -ENOMEM;
4169         }
4170         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4171                 printk(KERN_ERR
4172                         "intel_iommu_domain_init() failed\n");
4173                 domain_exit(dmar_domain);
4174                 return -ENOMEM;
4175         }
4176         domain_update_iommu_cap(dmar_domain);
4177         domain->priv = dmar_domain;
4178
4179         domain->geometry.aperture_start = 0;
4180         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4181         domain->geometry.force_aperture = true;
4182
4183         return 0;
4184 }
4185
4186 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
4187 {
4188         struct dmar_domain *dmar_domain = domain->priv;
4189
4190         domain->priv = NULL;
4191         domain_exit(dmar_domain);
4192 }
4193
4194 static int intel_iommu_attach_device(struct iommu_domain *domain,
4195                                      struct device *dev)
4196 {
4197         struct dmar_domain *dmar_domain = domain->priv;
4198         struct intel_iommu *iommu;
4199         int addr_width;
4200         u8 bus, devfn;
4201
4202         /* normally dev is not mapped */
4203         if (unlikely(domain_context_mapped(dev))) {
4204                 struct dmar_domain *old_domain;
4205
4206                 old_domain = find_domain(dev);
4207                 if (old_domain) {
4208                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4209                             dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4210                                 domain_remove_one_dev_info(old_domain, dev);
4211                         else
4212                                 domain_remove_dev_info(old_domain);
4213                 }
4214         }
4215
4216         iommu = device_to_iommu(dev, &bus, &devfn);
4217         if (!iommu)
4218                 return -ENODEV;
4219
4220         /* check if this iommu agaw is sufficient for max mapped address */
4221         addr_width = agaw_to_width(iommu->agaw);
4222         if (addr_width > cap_mgaw(iommu->cap))
4223                 addr_width = cap_mgaw(iommu->cap);
4224
4225         if (dmar_domain->max_addr > (1LL << addr_width)) {
4226                 printk(KERN_ERR "%s: iommu width (%d) is not "
4227                        "sufficient for the mapped address (%llx)\n",
4228                        __func__, addr_width, dmar_domain->max_addr);
4229                 return -EFAULT;
4230         }
4231         dmar_domain->gaw = addr_width;
4232
4233         /*
4234          * Knock out extra levels of page tables if necessary
4235          */
4236         while (iommu->agaw < dmar_domain->agaw) {
4237                 struct dma_pte *pte;
4238
4239                 pte = dmar_domain->pgd;
4240                 if (dma_pte_present(pte)) {
4241                         dmar_domain->pgd = (struct dma_pte *)
4242                                 phys_to_virt(dma_pte_addr(pte));
4243                         free_pgtable_page(pte);
4244                 }
4245                 dmar_domain->agaw--;
4246         }
4247
4248         return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4249 }
4250
4251 static void intel_iommu_detach_device(struct iommu_domain *domain,
4252                                       struct device *dev)
4253 {
4254         struct dmar_domain *dmar_domain = domain->priv;
4255
4256         domain_remove_one_dev_info(dmar_domain, dev);
4257 }
4258
4259 static int intel_iommu_map(struct iommu_domain *domain,
4260                            unsigned long iova, phys_addr_t hpa,
4261                            size_t size, int iommu_prot)
4262 {
4263         struct dmar_domain *dmar_domain = domain->priv;
4264         u64 max_addr;
4265         int prot = 0;
4266         int ret;
4267
4268         if (iommu_prot & IOMMU_READ)
4269                 prot |= DMA_PTE_READ;
4270         if (iommu_prot & IOMMU_WRITE)
4271                 prot |= DMA_PTE_WRITE;
4272         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4273                 prot |= DMA_PTE_SNP;
4274
4275         max_addr = iova + size;
4276         if (dmar_domain->max_addr < max_addr) {
4277                 u64 end;
4278
4279                 /* check if minimum agaw is sufficient for mapped address */
4280                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4281                 if (end < max_addr) {
4282                         printk(KERN_ERR "%s: iommu width (%d) is not "
4283                                "sufficient for the mapped address (%llx)\n",
4284                                __func__, dmar_domain->gaw, max_addr);
4285                         return -EFAULT;
4286                 }
4287                 dmar_domain->max_addr = max_addr;
4288         }
4289         /* Round up size to next multiple of PAGE_SIZE, if it and
4290            the low bits of hpa would take us onto the next page */
4291         size = aligned_nrpages(hpa, size);
4292         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4293                                  hpa >> VTD_PAGE_SHIFT, size, prot);
4294         return ret;
4295 }
4296
4297 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4298                                 unsigned long iova, size_t size)
4299 {
4300         struct dmar_domain *dmar_domain = domain->priv;
4301         struct page *freelist = NULL;
4302         struct intel_iommu *iommu;
4303         unsigned long start_pfn, last_pfn;
4304         unsigned int npages;
4305         int iommu_id, num, ndomains, level = 0;
4306
4307         /* Cope with horrid API which requires us to unmap more than the
4308            size argument if it happens to be a large-page mapping. */
4309         if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4310                 BUG();
4311
4312         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4313                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4314
4315         start_pfn = iova >> VTD_PAGE_SHIFT;
4316         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4317
4318         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4319
4320         npages = last_pfn - start_pfn + 1;
4321
4322         for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4323                iommu = g_iommus[iommu_id];
4324
4325                /*
4326                 * find bit position of dmar_domain
4327                 */
4328                ndomains = cap_ndoms(iommu->cap);
4329                for_each_set_bit(num, iommu->domain_ids, ndomains) {
4330                        if (iommu->domains[num] == dmar_domain)
4331                                iommu_flush_iotlb_psi(iommu, num, start_pfn,
4332                                                      npages, !freelist, 0);
4333                }
4334
4335         }
4336
4337         dma_free_pagelist(freelist);
4338
4339         if (dmar_domain->max_addr == iova + size)
4340                 dmar_domain->max_addr = iova;
4341
4342         return size;
4343 }
4344
4345 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4346                                             dma_addr_t iova)
4347 {
4348         struct dmar_domain *dmar_domain = domain->priv;
4349         struct dma_pte *pte;
4350         int level = 0;
4351         u64 phys = 0;
4352
4353         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4354         if (pte)
4355                 phys = dma_pte_addr(pte);
4356
4357         return phys;
4358 }
4359
4360 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4361                                       unsigned long cap)
4362 {
4363         struct dmar_domain *dmar_domain = domain->priv;
4364
4365         if (cap == IOMMU_CAP_CACHE_COHERENCY)
4366                 return dmar_domain->iommu_snooping;
4367         if (cap == IOMMU_CAP_INTR_REMAP)
4368                 return irq_remapping_enabled;
4369
4370         return 0;
4371 }
4372
4373 #define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4374
4375 static int intel_iommu_add_device(struct device *dev)
4376 {
4377         struct pci_dev *pdev = to_pci_dev(dev);
4378         struct pci_dev *bridge, *dma_pdev = NULL;
4379         struct iommu_group *group;
4380         int ret;
4381         u8 bus, devfn;
4382
4383         if (!device_to_iommu(dev, &bus, &devfn))
4384                 return -ENODEV;
4385
4386         bridge = pci_find_upstream_pcie_bridge(pdev);
4387         if (bridge) {
4388                 if (pci_is_pcie(bridge))
4389                         dma_pdev = pci_get_domain_bus_and_slot(
4390                                                 pci_domain_nr(pdev->bus),
4391                                                 bridge->subordinate->number, 0);
4392                 if (!dma_pdev)
4393                         dma_pdev = pci_dev_get(bridge);
4394         } else
4395                 dma_pdev = pci_dev_get(pdev);
4396
4397         /* Account for quirked devices */
4398         swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4399
4400         /*
4401          * If it's a multifunction device that does not support our
4402          * required ACS flags, add to the same group as lowest numbered
4403          * function that also does not suport the required ACS flags.
4404          */
4405         if (dma_pdev->multifunction &&
4406             !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4407                 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4408
4409                 for (i = 0; i < 8; i++) {
4410                         struct pci_dev *tmp;
4411
4412                         tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4413                         if (!tmp)
4414                                 continue;
4415
4416                         if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4417                                 swap_pci_ref(&dma_pdev, tmp);
4418                                 break;
4419                         }
4420                         pci_dev_put(tmp);
4421                 }
4422         }
4423
4424         /*
4425          * Devices on the root bus go through the iommu.  If that's not us,
4426          * find the next upstream device and test ACS up to the root bus.
4427          * Finding the next device may require skipping virtual buses.
4428          */
4429         while (!pci_is_root_bus(dma_pdev->bus)) {
4430                 struct pci_bus *bus = dma_pdev->bus;
4431
4432                 while (!bus->self) {
4433                         if (!pci_is_root_bus(bus))
4434                                 bus = bus->parent;
4435                         else
4436                                 goto root_bus;
4437                 }
4438
4439                 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4440                         break;
4441
4442                 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4443         }
4444
4445 root_bus:
4446         group = iommu_group_get(&dma_pdev->dev);
4447         pci_dev_put(dma_pdev);
4448         if (!group) {
4449                 group = iommu_group_alloc();
4450                 if (IS_ERR(group))
4451                         return PTR_ERR(group);
4452         }
4453
4454         ret = iommu_group_add_device(group, dev);
4455
4456         iommu_group_put(group);
4457         return ret;
4458 }
4459
4460 static void intel_iommu_remove_device(struct device *dev)
4461 {
4462         iommu_group_remove_device(dev);
4463 }
4464
4465 static struct iommu_ops intel_iommu_ops = {
4466         .domain_init    = intel_iommu_domain_init,
4467         .domain_destroy = intel_iommu_domain_destroy,
4468         .attach_dev     = intel_iommu_attach_device,
4469         .detach_dev     = intel_iommu_detach_device,
4470         .map            = intel_iommu_map,
4471         .unmap          = intel_iommu_unmap,
4472         .iova_to_phys   = intel_iommu_iova_to_phys,
4473         .domain_has_cap = intel_iommu_domain_has_cap,
4474         .add_device     = intel_iommu_add_device,
4475         .remove_device  = intel_iommu_remove_device,
4476         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
4477 };
4478
4479 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4480 {
4481         /* G4x/GM45 integrated gfx dmar support is totally busted. */
4482         printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4483         dmar_map_gfx = 0;
4484 }
4485
4486 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4487 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4488 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4489 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4490 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4491 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4492 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4493
4494 static void quirk_iommu_rwbf(struct pci_dev *dev)
4495 {
4496         /*
4497          * Mobile 4 Series Chipset neglects to set RWBF capability,
4498          * but needs it. Same seems to hold for the desktop versions.
4499          */
4500         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4501         rwbf_quirk = 1;
4502 }
4503
4504 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4505 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4506 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4507 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4508 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4509 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4510 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4511
4512 #define GGC 0x52
4513 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
4514 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
4515 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
4516 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
4517 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
4518 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
4519 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
4520 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
4521
4522 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4523 {
4524         unsigned short ggc;
4525
4526         if (pci_read_config_word(dev, GGC, &ggc))
4527                 return;
4528
4529         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4530                 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4531                 dmar_map_gfx = 0;
4532         } else if (dmar_map_gfx) {
4533                 /* we have to ensure the gfx device is idle before we flush */
4534                 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4535                 intel_iommu_strict = 1;
4536        }
4537 }
4538 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4539 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4540 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4541 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4542
4543 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4544    ISOCH DMAR unit for the Azalia sound device, but not give it any
4545    TLB entries, which causes it to deadlock. Check for that.  We do
4546    this in a function called from init_dmars(), instead of in a PCI
4547    quirk, because we don't want to print the obnoxious "BIOS broken"
4548    message if VT-d is actually disabled.
4549 */
4550 static void __init check_tylersburg_isoch(void)
4551 {
4552         struct pci_dev *pdev;
4553         uint32_t vtisochctrl;
4554
4555         /* If there's no Azalia in the system anyway, forget it. */
4556         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4557         if (!pdev)
4558                 return;
4559         pci_dev_put(pdev);
4560
4561         /* System Management Registers. Might be hidden, in which case
4562            we can't do the sanity check. But that's OK, because the
4563            known-broken BIOSes _don't_ actually hide it, so far. */
4564         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4565         if (!pdev)
4566                 return;
4567
4568         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4569                 pci_dev_put(pdev);
4570                 return;
4571         }
4572
4573         pci_dev_put(pdev);
4574
4575         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4576         if (vtisochctrl & 1)
4577                 return;
4578
4579         /* Drop all bits other than the number of TLB entries */
4580         vtisochctrl &= 0x1c;
4581
4582         /* If we have the recommended number of TLB entries (16), fine. */
4583         if (vtisochctrl == 0x10)
4584                 return;
4585
4586         /* Zero TLB entries? You get to ride the short bus to school. */
4587         if (!vtisochctrl) {
4588                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4589                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4590                      dmi_get_system_info(DMI_BIOS_VENDOR),
4591                      dmi_get_system_info(DMI_BIOS_VERSION),
4592                      dmi_get_system_info(DMI_PRODUCT_VERSION));
4593                 iommu_identity_mapping |= IDENTMAP_AZALIA;
4594                 return;
4595         }
4596
4597         printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4598                vtisochctrl);
4599 }