2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <asm/irq_remapping.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
46 #include "irq_remapping.h"
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
74 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
75 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
76 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
78 /* page table handling */
79 #define LEVEL_STRIDE (9)
80 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
98 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
100 static inline int agaw_to_level(int agaw)
105 static inline int agaw_to_width(int agaw)
107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
110 static inline int width_to_agaw(int width)
112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
115 static inline unsigned int level_to_offset_bits(int level)
117 return (level - 1) * LEVEL_STRIDE;
120 static inline int pfn_level_offset(unsigned long pfn, int level)
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
125 static inline unsigned long level_mask(int level)
127 return -1UL << level_to_offset_bits(level);
130 static inline unsigned long level_size(int level)
132 return 1UL << level_to_offset_bits(level);
135 static inline unsigned long align_to_level(unsigned long pfn, int level)
137 return (pfn + level_size(level) - 1) & level_mask(level);
140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
156 static inline unsigned long page_to_dma_pfn(struct page *pg)
158 return mm_to_dma_pfn(page_to_pfn(pg));
160 static inline unsigned long virt_to_dma_pfn(void *p)
162 return page_to_dma_pfn(virt_to_page(p));
165 /* global iommu list, set NULL for ignored DMAR units */
166 static struct intel_iommu **g_iommus;
168 static void __init check_tylersburg_isoch(void);
169 static int rwbf_quirk;
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
175 static int force_on = 0;
180 * 12-63: Context Ptr (12 - (haw-1))
187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188 static inline bool root_present(struct root_entry *root)
190 return (root->val & 1);
192 static inline void set_root_present(struct root_entry *root)
196 static inline void set_root_value(struct root_entry *root, unsigned long value)
198 root->val |= value & VTD_PAGE_MASK;
201 static inline struct context_entry *
202 get_context_addr_from_root(struct root_entry *root)
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
221 struct context_entry {
226 static inline bool context_present(struct context_entry *context)
228 return (context->lo & 1);
230 static inline void context_set_present(struct context_entry *context)
235 static inline void context_set_fault_enable(struct context_entry *context)
237 context->lo &= (((u64)-1) << 2) | 1;
240 static inline void context_set_translation_type(struct context_entry *context,
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
247 static inline void context_set_address_root(struct context_entry *context,
250 context->lo |= value & VTD_PAGE_MASK;
253 static inline void context_set_address_width(struct context_entry *context,
256 context->hi |= value & 7;
259 static inline void context_set_domain_id(struct context_entry *context,
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
265 static inline void context_clear_entry(struct context_entry *context)
278 * 12-63: Host physcial address
284 static inline void dma_clear_pte(struct dma_pte *pte)
289 static inline u64 dma_pte_addr(struct dma_pte *pte)
292 return pte->val & VTD_PAGE_MASK;
294 /* Must have a full atomic 64-bit read */
295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
299 static inline bool dma_pte_present(struct dma_pte *pte)
301 return (pte->val & 3) != 0;
304 static inline bool dma_pte_superpage(struct dma_pte *pte)
306 return (pte->val & (1 << 7));
309 static inline int first_pte_in_page(struct dma_pte *pte)
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
320 static struct dmar_domain *si_domain;
321 static int hw_pass_through = 1;
323 /* devices under the same p2p bridge are owned in one domain */
324 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326 /* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
329 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
331 /* si_domain contains mulitple devices */
332 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
334 /* define the limit of IOMMUs supported in each domain */
336 # define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
338 # define IOMMU_UNITS_SUPPORTED 64
342 int id; /* domain id */
343 int nid; /* node id */
344 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
345 /* bitmap of iommus this domain uses*/
347 struct list_head devices; /* all devices' list */
348 struct iova_domain iovad; /* iova's that belong to this domain */
350 struct dma_pte *pgd; /* virtual address */
351 int gaw; /* max guest address width */
353 /* adjusted guest address width, 0 is level 2 30-bit */
356 int flags; /* flags to find out type of domain */
358 int iommu_coherency;/* indicate coherency of iommu access */
359 int iommu_snooping; /* indicate snooping control feature*/
360 int iommu_count; /* reference count of iommu */
361 int iommu_superpage;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
364 spinlock_t iommu_lock; /* protect iommu set in domain */
365 u64 max_addr; /* maximum mapped address */
368 /* PCI domain-device relationship */
369 struct device_domain_info {
370 struct list_head link; /* link to domain siblings */
371 struct list_head global; /* link to global list */
372 u8 bus; /* PCI bus number */
373 u8 devfn; /* PCI devfn number */
374 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
375 struct intel_iommu *iommu; /* IOMMU used by this device */
376 struct dmar_domain *domain; /* pointer to domain */
379 struct dmar_rmrr_unit {
380 struct list_head list; /* list of rmrr units */
381 struct acpi_dmar_header *hdr; /* ACPI header */
382 u64 base_address; /* reserved base address*/
383 u64 end_address; /* reserved end address */
384 struct dmar_dev_scope *devices; /* target devices */
385 int devices_cnt; /* target device count */
388 struct dmar_atsr_unit {
389 struct list_head list; /* list of ATSR units */
390 struct acpi_dmar_header *hdr; /* ACPI header */
391 struct dmar_dev_scope *devices; /* target devices */
392 int devices_cnt; /* target device count */
393 u8 include_all:1; /* include all ports */
396 static LIST_HEAD(dmar_atsr_units);
397 static LIST_HEAD(dmar_rmrr_units);
399 #define for_each_rmrr_units(rmrr) \
400 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
402 static void flush_unmaps_timeout(unsigned long data);
404 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
406 #define HIGH_WATER_MARK 250
407 struct deferred_flush_tables {
409 struct iova *iova[HIGH_WATER_MARK];
410 struct dmar_domain *domain[HIGH_WATER_MARK];
411 struct page *freelist[HIGH_WATER_MARK];
414 static struct deferred_flush_tables *deferred_flush;
416 /* bitmap for indexing intel_iommus */
417 static int g_num_of_iommus;
419 static DEFINE_SPINLOCK(async_umap_flush_lock);
420 static LIST_HEAD(unmaps_to_do);
423 static long list_size;
425 static void domain_exit(struct dmar_domain *domain);
426 static void domain_remove_dev_info(struct dmar_domain *domain);
427 static void domain_remove_one_dev_info(struct dmar_domain *domain,
429 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
432 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
433 int dmar_disabled = 0;
435 int dmar_disabled = 1;
436 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
438 int intel_iommu_enabled = 0;
439 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
441 static int dmar_map_gfx = 1;
442 static int dmar_forcedac;
443 static int intel_iommu_strict;
444 static int intel_iommu_superpage = 1;
446 int intel_iommu_gfx_mapped;
447 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
449 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
450 static DEFINE_SPINLOCK(device_domain_lock);
451 static LIST_HEAD(device_domain_list);
453 static struct iommu_ops intel_iommu_ops;
455 static int __init intel_iommu_setup(char *str)
460 if (!strncmp(str, "on", 2)) {
462 printk(KERN_INFO "Intel-IOMMU: enabled\n");
463 } else if (!strncmp(str, "off", 3)) {
465 printk(KERN_INFO "Intel-IOMMU: disabled\n");
466 } else if (!strncmp(str, "igfx_off", 8)) {
469 "Intel-IOMMU: disable GFX device mapping\n");
470 } else if (!strncmp(str, "forcedac", 8)) {
472 "Intel-IOMMU: Forcing DAC for PCI devices\n");
474 } else if (!strncmp(str, "strict", 6)) {
476 "Intel-IOMMU: disable batched IOTLB flush\n");
477 intel_iommu_strict = 1;
478 } else if (!strncmp(str, "sp_off", 6)) {
480 "Intel-IOMMU: disable supported super page\n");
481 intel_iommu_superpage = 0;
484 str += strcspn(str, ",");
490 __setup("intel_iommu=", intel_iommu_setup);
492 static struct kmem_cache *iommu_domain_cache;
493 static struct kmem_cache *iommu_devinfo_cache;
494 static struct kmem_cache *iommu_iova_cache;
496 static inline void *alloc_pgtable_page(int node)
501 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
503 vaddr = page_address(page);
507 static inline void free_pgtable_page(void *vaddr)
509 free_page((unsigned long)vaddr);
512 static inline void *alloc_domain_mem(void)
514 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
517 static void free_domain_mem(void *vaddr)
519 kmem_cache_free(iommu_domain_cache, vaddr);
522 static inline void * alloc_devinfo_mem(void)
524 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
527 static inline void free_devinfo_mem(void *vaddr)
529 kmem_cache_free(iommu_devinfo_cache, vaddr);
532 struct iova *alloc_iova_mem(void)
534 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
537 void free_iova_mem(struct iova *iova)
539 kmem_cache_free(iommu_iova_cache, iova);
543 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
548 sagaw = cap_sagaw(iommu->cap);
549 for (agaw = width_to_agaw(max_gaw);
551 if (test_bit(agaw, &sagaw))
559 * Calculate max SAGAW for each iommu.
561 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
563 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
567 * calculate agaw for each iommu.
568 * "SAGAW" may be different across iommus, use a default agaw, and
569 * get a supported less agaw for iommus that don't support the default agaw.
571 int iommu_calculate_agaw(struct intel_iommu *iommu)
573 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
576 /* This functionin only returns single iommu in a domain */
577 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
581 /* si_domain and vm domain should not get here. */
582 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
583 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
585 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
586 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
589 return g_iommus[iommu_id];
592 static void domain_update_iommu_coherency(struct dmar_domain *domain)
594 struct dmar_drhd_unit *drhd;
595 struct intel_iommu *iommu;
598 domain->iommu_coherency = 1;
600 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
602 if (!ecap_coherent(g_iommus[i]->ecap)) {
603 domain->iommu_coherency = 0;
610 /* No hardware attached; use lowest common denominator */
612 for_each_active_iommu(iommu, drhd) {
613 if (!ecap_coherent(iommu->ecap)) {
614 domain->iommu_coherency = 0;
621 static void domain_update_iommu_snooping(struct dmar_domain *domain)
625 domain->iommu_snooping = 1;
627 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
628 if (!ecap_sc_support(g_iommus[i]->ecap)) {
629 domain->iommu_snooping = 0;
635 static void domain_update_iommu_superpage(struct dmar_domain *domain)
637 struct dmar_drhd_unit *drhd;
638 struct intel_iommu *iommu = NULL;
641 if (!intel_iommu_superpage) {
642 domain->iommu_superpage = 0;
646 /* set iommu_superpage to the smallest common denominator */
648 for_each_active_iommu(iommu, drhd) {
649 mask &= cap_super_page_val(iommu->cap);
656 domain->iommu_superpage = fls(mask);
659 /* Some capabilities may be different across iommus */
660 static void domain_update_iommu_cap(struct dmar_domain *domain)
662 domain_update_iommu_coherency(domain);
663 domain_update_iommu_snooping(domain);
664 domain_update_iommu_superpage(domain);
667 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
669 struct dmar_drhd_unit *drhd = NULL;
670 struct intel_iommu *iommu;
672 struct pci_dev *ptmp, *pdev = NULL;
676 if (dev_is_pci(dev)) {
677 pdev = to_pci_dev(dev);
678 segment = pci_domain_nr(pdev->bus);
679 } else if (ACPI_COMPANION(dev))
680 dev = &ACPI_COMPANION(dev)->dev;
683 for_each_active_iommu(iommu, drhd) {
684 if (pdev && segment != drhd->segment)
687 for_each_active_dev_scope(drhd->devices,
688 drhd->devices_cnt, i, tmp) {
690 *bus = drhd->devices[i].bus;
691 *devfn = drhd->devices[i].devfn;
695 if (!pdev || !dev_is_pci(tmp))
698 ptmp = to_pci_dev(tmp);
699 if (ptmp->subordinate &&
700 ptmp->subordinate->number <= pdev->bus->number &&
701 ptmp->subordinate->busn_res.end >= pdev->bus->number)
705 if (pdev && drhd->include_all) {
707 *bus = pdev->bus->number;
708 *devfn = pdev->devfn;
719 static void domain_flush_cache(struct dmar_domain *domain,
720 void *addr, int size)
722 if (!domain->iommu_coherency)
723 clflush_cache_range(addr, size);
726 /* Gets context entry for a given bus and devfn */
727 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
730 struct root_entry *root;
731 struct context_entry *context;
732 unsigned long phy_addr;
735 spin_lock_irqsave(&iommu->lock, flags);
736 root = &iommu->root_entry[bus];
737 context = get_context_addr_from_root(root);
739 context = (struct context_entry *)
740 alloc_pgtable_page(iommu->node);
742 spin_unlock_irqrestore(&iommu->lock, flags);
745 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
746 phy_addr = virt_to_phys((void *)context);
747 set_root_value(root, phy_addr);
748 set_root_present(root);
749 __iommu_flush_cache(iommu, root, sizeof(*root));
751 spin_unlock_irqrestore(&iommu->lock, flags);
752 return &context[devfn];
755 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
757 struct root_entry *root;
758 struct context_entry *context;
762 spin_lock_irqsave(&iommu->lock, flags);
763 root = &iommu->root_entry[bus];
764 context = get_context_addr_from_root(root);
769 ret = context_present(&context[devfn]);
771 spin_unlock_irqrestore(&iommu->lock, flags);
775 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
777 struct root_entry *root;
778 struct context_entry *context;
781 spin_lock_irqsave(&iommu->lock, flags);
782 root = &iommu->root_entry[bus];
783 context = get_context_addr_from_root(root);
785 context_clear_entry(&context[devfn]);
786 __iommu_flush_cache(iommu, &context[devfn], \
789 spin_unlock_irqrestore(&iommu->lock, flags);
792 static void free_context_table(struct intel_iommu *iommu)
794 struct root_entry *root;
797 struct context_entry *context;
799 spin_lock_irqsave(&iommu->lock, flags);
800 if (!iommu->root_entry) {
803 for (i = 0; i < ROOT_ENTRY_NR; i++) {
804 root = &iommu->root_entry[i];
805 context = get_context_addr_from_root(root);
807 free_pgtable_page(context);
809 free_pgtable_page(iommu->root_entry);
810 iommu->root_entry = NULL;
812 spin_unlock_irqrestore(&iommu->lock, flags);
815 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
816 unsigned long pfn, int *target_level)
818 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
819 struct dma_pte *parent, *pte = NULL;
820 int level = agaw_to_level(domain->agaw);
823 BUG_ON(!domain->pgd);
825 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
826 /* Address beyond IOMMU's addressing capabilities. */
829 parent = domain->pgd;
834 offset = pfn_level_offset(pfn, level);
835 pte = &parent[offset];
836 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
838 if (level == *target_level)
841 if (!dma_pte_present(pte)) {
844 tmp_page = alloc_pgtable_page(domain->nid);
849 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
850 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
851 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
852 /* Someone else set it while we were thinking; use theirs. */
853 free_pgtable_page(tmp_page);
856 domain_flush_cache(domain, pte, sizeof(*pte));
862 parent = phys_to_virt(dma_pte_addr(pte));
867 *target_level = level;
873 /* return address's pte at specific level */
874 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
876 int level, int *large_page)
878 struct dma_pte *parent, *pte = NULL;
879 int total = agaw_to_level(domain->agaw);
882 parent = domain->pgd;
883 while (level <= total) {
884 offset = pfn_level_offset(pfn, total);
885 pte = &parent[offset];
889 if (!dma_pte_present(pte)) {
894 if (pte->val & DMA_PTE_LARGE_PAGE) {
899 parent = phys_to_virt(dma_pte_addr(pte));
905 /* clear last level pte, a tlb flush should be followed */
906 static void dma_pte_clear_range(struct dmar_domain *domain,
907 unsigned long start_pfn,
908 unsigned long last_pfn)
910 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
911 unsigned int large_page = 1;
912 struct dma_pte *first_pte, *pte;
914 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
915 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
916 BUG_ON(start_pfn > last_pfn);
918 /* we don't need lock here; nobody else touches the iova range */
921 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
923 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
928 start_pfn += lvl_to_nr_pages(large_page);
930 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
932 domain_flush_cache(domain, first_pte,
933 (void *)pte - (void *)first_pte);
935 } while (start_pfn && start_pfn <= last_pfn);
938 static void dma_pte_free_level(struct dmar_domain *domain, int level,
939 struct dma_pte *pte, unsigned long pfn,
940 unsigned long start_pfn, unsigned long last_pfn)
942 pfn = max(start_pfn, pfn);
943 pte = &pte[pfn_level_offset(pfn, level)];
946 unsigned long level_pfn;
947 struct dma_pte *level_pte;
949 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
952 level_pfn = pfn & level_mask(level - 1);
953 level_pte = phys_to_virt(dma_pte_addr(pte));
956 dma_pte_free_level(domain, level - 1, level_pte,
957 level_pfn, start_pfn, last_pfn);
959 /* If range covers entire pagetable, free it */
960 if (!(start_pfn > level_pfn ||
961 last_pfn < level_pfn + level_size(level) - 1)) {
963 domain_flush_cache(domain, pte, sizeof(*pte));
964 free_pgtable_page(level_pte);
967 pfn += level_size(level);
968 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
971 /* free page table pages. last level pte should already be cleared */
972 static void dma_pte_free_pagetable(struct dmar_domain *domain,
973 unsigned long start_pfn,
974 unsigned long last_pfn)
976 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
978 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
979 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
980 BUG_ON(start_pfn > last_pfn);
982 /* We don't need lock here; nobody else touches the iova range */
983 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
984 domain->pgd, 0, start_pfn, last_pfn);
987 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
988 free_pgtable_page(domain->pgd);
993 /* When a page at a given level is being unlinked from its parent, we don't
994 need to *modify* it at all. All we need to do is make a list of all the
995 pages which can be freed just as soon as we've flushed the IOTLB and we
996 know the hardware page-walk will no longer touch them.
997 The 'pte' argument is the *parent* PTE, pointing to the page that is to
999 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1000 int level, struct dma_pte *pte,
1001 struct page *freelist)
1005 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1006 pg->freelist = freelist;
1012 for (pte = page_address(pg); !first_pte_in_page(pte); pte++) {
1013 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1014 freelist = dma_pte_list_pagetables(domain, level - 1,
1021 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1022 struct dma_pte *pte, unsigned long pfn,
1023 unsigned long start_pfn,
1024 unsigned long last_pfn,
1025 struct page *freelist)
1027 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1029 pfn = max(start_pfn, pfn);
1030 pte = &pte[pfn_level_offset(pfn, level)];
1033 unsigned long level_pfn;
1035 if (!dma_pte_present(pte))
1038 level_pfn = pfn & level_mask(level);
1040 /* If range covers entire pagetable, free it */
1041 if (start_pfn <= level_pfn &&
1042 last_pfn >= level_pfn + level_size(level) - 1) {
1043 /* These suborbinate page tables are going away entirely. Don't
1044 bother to clear them; we're just going to *free* them. */
1045 if (level > 1 && !dma_pte_superpage(pte))
1046 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1052 } else if (level > 1) {
1053 /* Recurse down into a level that isn't *entirely* obsolete */
1054 freelist = dma_pte_clear_level(domain, level - 1,
1055 phys_to_virt(dma_pte_addr(pte)),
1056 level_pfn, start_pfn, last_pfn,
1060 pfn += level_size(level);
1061 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1064 domain_flush_cache(domain, first_pte,
1065 (void *)++last_pte - (void *)first_pte);
1070 /* We can't just free the pages because the IOMMU may still be walking
1071 the page tables, and may have cached the intermediate levels. The
1072 pages can only be freed after the IOTLB flush has been done. */
1073 struct page *domain_unmap(struct dmar_domain *domain,
1074 unsigned long start_pfn,
1075 unsigned long last_pfn)
1077 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1078 struct page *freelist = NULL;
1080 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1081 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1082 BUG_ON(start_pfn > last_pfn);
1084 /* we don't need lock here; nobody else touches the iova range */
1085 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1086 domain->pgd, 0, start_pfn, last_pfn, NULL);
1089 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1090 struct page *pgd_page = virt_to_page(domain->pgd);
1091 pgd_page->freelist = freelist;
1092 freelist = pgd_page;
1100 void dma_free_pagelist(struct page *freelist)
1104 while ((pg = freelist)) {
1105 freelist = pg->freelist;
1106 free_pgtable_page(page_address(pg));
1110 /* iommu handling */
1111 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1113 struct root_entry *root;
1114 unsigned long flags;
1116 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1120 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1122 spin_lock_irqsave(&iommu->lock, flags);
1123 iommu->root_entry = root;
1124 spin_unlock_irqrestore(&iommu->lock, flags);
1129 static void iommu_set_root_entry(struct intel_iommu *iommu)
1135 addr = iommu->root_entry;
1137 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1138 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1140 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1142 /* Make sure hardware complete it */
1143 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1144 readl, (sts & DMA_GSTS_RTPS), sts);
1146 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1149 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1154 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1157 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1158 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1160 /* Make sure hardware complete it */
1161 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1162 readl, (!(val & DMA_GSTS_WBFS)), val);
1164 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1167 /* return value determine if we need a write buffer flush */
1168 static void __iommu_flush_context(struct intel_iommu *iommu,
1169 u16 did, u16 source_id, u8 function_mask,
1176 case DMA_CCMD_GLOBAL_INVL:
1177 val = DMA_CCMD_GLOBAL_INVL;
1179 case DMA_CCMD_DOMAIN_INVL:
1180 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1182 case DMA_CCMD_DEVICE_INVL:
1183 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1184 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1189 val |= DMA_CCMD_ICC;
1191 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1192 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1194 /* Make sure hardware complete it */
1195 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1196 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1198 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1201 /* return value determine if we need a write buffer flush */
1202 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1203 u64 addr, unsigned int size_order, u64 type)
1205 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1206 u64 val = 0, val_iva = 0;
1210 case DMA_TLB_GLOBAL_FLUSH:
1211 /* global flush doesn't need set IVA_REG */
1212 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1214 case DMA_TLB_DSI_FLUSH:
1215 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1217 case DMA_TLB_PSI_FLUSH:
1218 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1219 /* IH bit is passed in as part of address */
1220 val_iva = size_order | addr;
1225 /* Note: set drain read/write */
1228 * This is probably to be super secure.. Looks like we can
1229 * ignore it without any impact.
1231 if (cap_read_drain(iommu->cap))
1232 val |= DMA_TLB_READ_DRAIN;
1234 if (cap_write_drain(iommu->cap))
1235 val |= DMA_TLB_WRITE_DRAIN;
1237 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1238 /* Note: Only uses first TLB reg currently */
1240 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1241 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1243 /* Make sure hardware complete it */
1244 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1245 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1247 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1249 /* check IOTLB invalidation granularity */
1250 if (DMA_TLB_IAIG(val) == 0)
1251 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1252 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1253 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1254 (unsigned long long)DMA_TLB_IIRG(type),
1255 (unsigned long long)DMA_TLB_IAIG(val));
1258 static struct device_domain_info *
1259 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1263 unsigned long flags;
1264 struct device_domain_info *info;
1265 struct pci_dev *pdev;
1267 if (!ecap_dev_iotlb_support(iommu->ecap))
1273 spin_lock_irqsave(&device_domain_lock, flags);
1274 list_for_each_entry(info, &domain->devices, link)
1275 if (info->bus == bus && info->devfn == devfn) {
1279 spin_unlock_irqrestore(&device_domain_lock, flags);
1281 if (!found || !info->dev || !dev_is_pci(info->dev))
1284 pdev = to_pci_dev(info->dev);
1286 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1289 if (!dmar_find_matched_atsr_unit(pdev))
1295 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1297 if (!info || !dev_is_pci(info->dev))
1300 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1303 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1305 if (!info->dev || !dev_is_pci(info->dev) ||
1306 !pci_ats_enabled(to_pci_dev(info->dev)))
1309 pci_disable_ats(to_pci_dev(info->dev));
1312 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1313 u64 addr, unsigned mask)
1316 unsigned long flags;
1317 struct device_domain_info *info;
1319 spin_lock_irqsave(&device_domain_lock, flags);
1320 list_for_each_entry(info, &domain->devices, link) {
1321 struct pci_dev *pdev;
1322 if (!info->dev || !dev_is_pci(info->dev))
1325 pdev = to_pci_dev(info->dev);
1326 if (!pci_ats_enabled(pdev))
1329 sid = info->bus << 8 | info->devfn;
1330 qdep = pci_ats_queue_depth(pdev);
1331 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1333 spin_unlock_irqrestore(&device_domain_lock, flags);
1336 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1337 unsigned long pfn, unsigned int pages, int ih, int map)
1339 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1340 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1347 * Fallback to domain selective flush if no PSI support or the size is
1349 * PSI requires page size to be 2 ^ x, and the base address is naturally
1350 * aligned to the size
1352 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1353 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1356 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1360 * In caching mode, changes of pages from non-present to present require
1361 * flush. However, device IOTLB doesn't need to be flushed in this case.
1363 if (!cap_caching_mode(iommu->cap) || !map)
1364 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1367 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1370 unsigned long flags;
1372 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1373 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1374 pmen &= ~DMA_PMEN_EPM;
1375 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1377 /* wait for the protected region status bit to clear */
1378 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1379 readl, !(pmen & DMA_PMEN_PRS), pmen);
1381 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1384 static int iommu_enable_translation(struct intel_iommu *iommu)
1387 unsigned long flags;
1389 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1390 iommu->gcmd |= DMA_GCMD_TE;
1391 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1393 /* Make sure hardware complete it */
1394 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1395 readl, (sts & DMA_GSTS_TES), sts);
1397 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1401 static int iommu_disable_translation(struct intel_iommu *iommu)
1406 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1407 iommu->gcmd &= ~DMA_GCMD_TE;
1408 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1410 /* Make sure hardware complete it */
1411 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1412 readl, (!(sts & DMA_GSTS_TES)), sts);
1414 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1419 static int iommu_init_domains(struct intel_iommu *iommu)
1421 unsigned long ndomains;
1422 unsigned long nlongs;
1424 ndomains = cap_ndoms(iommu->cap);
1425 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1426 iommu->seq_id, ndomains);
1427 nlongs = BITS_TO_LONGS(ndomains);
1429 spin_lock_init(&iommu->lock);
1431 /* TBD: there might be 64K domains,
1432 * consider other allocation for future chip
1434 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1435 if (!iommu->domain_ids) {
1436 pr_err("IOMMU%d: allocating domain id array failed\n",
1440 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1442 if (!iommu->domains) {
1443 pr_err("IOMMU%d: allocating domain array failed\n",
1445 kfree(iommu->domain_ids);
1446 iommu->domain_ids = NULL;
1451 * if Caching mode is set, then invalid translations are tagged
1452 * with domainid 0. Hence we need to pre-allocate it.
1454 if (cap_caching_mode(iommu->cap))
1455 set_bit(0, iommu->domain_ids);
1459 static void free_dmar_iommu(struct intel_iommu *iommu)
1461 struct dmar_domain *domain;
1463 unsigned long flags;
1465 if ((iommu->domains) && (iommu->domain_ids)) {
1466 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1468 * Domain id 0 is reserved for invalid translation
1469 * if hardware supports caching mode.
1471 if (cap_caching_mode(iommu->cap) && i == 0)
1474 domain = iommu->domains[i];
1475 clear_bit(i, iommu->domain_ids);
1477 spin_lock_irqsave(&domain->iommu_lock, flags);
1478 count = --domain->iommu_count;
1479 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1481 domain_exit(domain);
1485 if (iommu->gcmd & DMA_GCMD_TE)
1486 iommu_disable_translation(iommu);
1488 kfree(iommu->domains);
1489 kfree(iommu->domain_ids);
1490 iommu->domains = NULL;
1491 iommu->domain_ids = NULL;
1493 g_iommus[iommu->seq_id] = NULL;
1495 /* free context mapping */
1496 free_context_table(iommu);
1499 static struct dmar_domain *alloc_domain(bool vm)
1501 /* domain id for virtual machine, it won't be set in context */
1502 static atomic_t vm_domid = ATOMIC_INIT(0);
1503 struct dmar_domain *domain;
1505 domain = alloc_domain_mem();
1510 domain->iommu_count = 0;
1511 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1513 spin_lock_init(&domain->iommu_lock);
1514 INIT_LIST_HEAD(&domain->devices);
1516 domain->id = atomic_inc_return(&vm_domid);
1517 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1523 static int iommu_attach_domain(struct dmar_domain *domain,
1524 struct intel_iommu *iommu)
1527 unsigned long ndomains;
1528 unsigned long flags;
1530 ndomains = cap_ndoms(iommu->cap);
1532 spin_lock_irqsave(&iommu->lock, flags);
1534 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1535 if (num >= ndomains) {
1536 spin_unlock_irqrestore(&iommu->lock, flags);
1537 printk(KERN_ERR "IOMMU: no free domain ids\n");
1542 domain->iommu_count++;
1543 set_bit(num, iommu->domain_ids);
1544 set_bit(iommu->seq_id, domain->iommu_bmp);
1545 iommu->domains[num] = domain;
1546 spin_unlock_irqrestore(&iommu->lock, flags);
1551 static void iommu_detach_domain(struct dmar_domain *domain,
1552 struct intel_iommu *iommu)
1554 unsigned long flags;
1557 spin_lock_irqsave(&iommu->lock, flags);
1558 ndomains = cap_ndoms(iommu->cap);
1559 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1560 if (iommu->domains[num] == domain) {
1561 clear_bit(num, iommu->domain_ids);
1562 iommu->domains[num] = NULL;
1566 spin_unlock_irqrestore(&iommu->lock, flags);
1569 static struct iova_domain reserved_iova_list;
1570 static struct lock_class_key reserved_rbtree_key;
1572 static int dmar_init_reserved_ranges(void)
1574 struct pci_dev *pdev = NULL;
1578 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1580 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1581 &reserved_rbtree_key);
1583 /* IOAPIC ranges shouldn't be accessed by DMA */
1584 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1585 IOVA_PFN(IOAPIC_RANGE_END));
1587 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1591 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1592 for_each_pci_dev(pdev) {
1595 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1596 r = &pdev->resource[i];
1597 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1599 iova = reserve_iova(&reserved_iova_list,
1603 printk(KERN_ERR "Reserve iova failed\n");
1611 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1613 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1616 static inline int guestwidth_to_adjustwidth(int gaw)
1619 int r = (gaw - 12) % 9;
1630 static int domain_init(struct dmar_domain *domain, int guest_width)
1632 struct intel_iommu *iommu;
1633 int adjust_width, agaw;
1634 unsigned long sagaw;
1636 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1637 domain_reserve_special_ranges(domain);
1639 /* calculate AGAW */
1640 iommu = domain_get_iommu(domain);
1641 if (guest_width > cap_mgaw(iommu->cap))
1642 guest_width = cap_mgaw(iommu->cap);
1643 domain->gaw = guest_width;
1644 adjust_width = guestwidth_to_adjustwidth(guest_width);
1645 agaw = width_to_agaw(adjust_width);
1646 sagaw = cap_sagaw(iommu->cap);
1647 if (!test_bit(agaw, &sagaw)) {
1648 /* hardware doesn't support it, choose a bigger one */
1649 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1650 agaw = find_next_bit(&sagaw, 5, agaw);
1654 domain->agaw = agaw;
1656 if (ecap_coherent(iommu->ecap))
1657 domain->iommu_coherency = 1;
1659 domain->iommu_coherency = 0;
1661 if (ecap_sc_support(iommu->ecap))
1662 domain->iommu_snooping = 1;
1664 domain->iommu_snooping = 0;
1666 if (intel_iommu_superpage)
1667 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1669 domain->iommu_superpage = 0;
1671 domain->nid = iommu->node;
1673 /* always allocate the top pgd */
1674 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1677 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1681 static void domain_exit(struct dmar_domain *domain)
1683 struct dmar_drhd_unit *drhd;
1684 struct intel_iommu *iommu;
1685 struct page *freelist = NULL;
1687 /* Domain 0 is reserved, so dont process it */
1691 /* Flush any lazy unmaps that may reference this domain */
1692 if (!intel_iommu_strict)
1693 flush_unmaps_timeout(0);
1695 /* remove associated devices */
1696 domain_remove_dev_info(domain);
1699 put_iova_domain(&domain->iovad);
1701 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1703 /* clear attached or cached domains */
1705 for_each_active_iommu(iommu, drhd)
1706 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1707 test_bit(iommu->seq_id, domain->iommu_bmp))
1708 iommu_detach_domain(domain, iommu);
1711 dma_free_pagelist(freelist);
1713 free_domain_mem(domain);
1716 static int domain_context_mapping_one(struct dmar_domain *domain,
1717 struct intel_iommu *iommu,
1718 u8 bus, u8 devfn, int translation)
1720 struct context_entry *context;
1721 unsigned long flags;
1722 struct dma_pte *pgd;
1724 unsigned long ndomains;
1727 struct device_domain_info *info = NULL;
1729 pr_debug("Set context mapping for %02x:%02x.%d\n",
1730 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1732 BUG_ON(!domain->pgd);
1733 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1734 translation != CONTEXT_TT_MULTI_LEVEL);
1736 context = device_to_context_entry(iommu, bus, devfn);
1739 spin_lock_irqsave(&iommu->lock, flags);
1740 if (context_present(context)) {
1741 spin_unlock_irqrestore(&iommu->lock, flags);
1748 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1749 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1752 /* find an available domain id for this device in iommu */
1753 ndomains = cap_ndoms(iommu->cap);
1754 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1755 if (iommu->domains[num] == domain) {
1763 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1764 if (num >= ndomains) {
1765 spin_unlock_irqrestore(&iommu->lock, flags);
1766 printk(KERN_ERR "IOMMU: no free domain ids\n");
1770 set_bit(num, iommu->domain_ids);
1771 iommu->domains[num] = domain;
1775 /* Skip top levels of page tables for
1776 * iommu which has less agaw than default.
1777 * Unnecessary for PT mode.
1779 if (translation != CONTEXT_TT_PASS_THROUGH) {
1780 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1781 pgd = phys_to_virt(dma_pte_addr(pgd));
1782 if (!dma_pte_present(pgd)) {
1783 spin_unlock_irqrestore(&iommu->lock, flags);
1790 context_set_domain_id(context, id);
1792 if (translation != CONTEXT_TT_PASS_THROUGH) {
1793 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1794 translation = info ? CONTEXT_TT_DEV_IOTLB :
1795 CONTEXT_TT_MULTI_LEVEL;
1798 * In pass through mode, AW must be programmed to indicate the largest
1799 * AGAW value supported by hardware. And ASR is ignored by hardware.
1801 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1802 context_set_address_width(context, iommu->msagaw);
1804 context_set_address_root(context, virt_to_phys(pgd));
1805 context_set_address_width(context, iommu->agaw);
1808 context_set_translation_type(context, translation);
1809 context_set_fault_enable(context);
1810 context_set_present(context);
1811 domain_flush_cache(domain, context, sizeof(*context));
1814 * It's a non-present to present mapping. If hardware doesn't cache
1815 * non-present entry we only need to flush the write-buffer. If the
1816 * _does_ cache non-present entries, then it does so in the special
1817 * domain #0, which we have to flush:
1819 if (cap_caching_mode(iommu->cap)) {
1820 iommu->flush.flush_context(iommu, 0,
1821 (((u16)bus) << 8) | devfn,
1822 DMA_CCMD_MASK_NOBIT,
1823 DMA_CCMD_DEVICE_INVL);
1824 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1826 iommu_flush_write_buffer(iommu);
1828 iommu_enable_dev_iotlb(info);
1829 spin_unlock_irqrestore(&iommu->lock, flags);
1831 spin_lock_irqsave(&domain->iommu_lock, flags);
1832 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1833 domain->iommu_count++;
1834 if (domain->iommu_count == 1)
1835 domain->nid = iommu->node;
1836 domain_update_iommu_cap(domain);
1838 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1843 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1847 struct pci_dev *pdev, *tmp, *parent;
1848 struct intel_iommu *iommu;
1851 iommu = device_to_iommu(dev, &bus, &devfn);
1855 ret = domain_context_mapping_one(domain, iommu, bus, devfn,
1857 if (ret || !dev_is_pci(dev))
1860 /* dependent device mapping */
1861 pdev = to_pci_dev(dev);
1862 tmp = pci_find_upstream_pcie_bridge(pdev);
1865 /* Secondary interface's bus number and devfn 0 */
1866 parent = pdev->bus->self;
1867 while (parent != tmp) {
1868 ret = domain_context_mapping_one(domain, iommu,
1869 parent->bus->number,
1870 parent->devfn, translation);
1873 parent = parent->bus->self;
1875 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1876 return domain_context_mapping_one(domain, iommu,
1877 tmp->subordinate->number, 0,
1879 else /* this is a legacy PCI bridge */
1880 return domain_context_mapping_one(domain, iommu,
1886 static int domain_context_mapped(struct device *dev)
1889 struct pci_dev *pdev, *tmp, *parent;
1890 struct intel_iommu *iommu;
1893 iommu = device_to_iommu(dev, &bus, &devfn);
1897 ret = device_context_mapped(iommu, bus, devfn);
1898 if (!ret || !dev_is_pci(dev))
1901 /* dependent device mapping */
1902 pdev = to_pci_dev(dev);
1903 tmp = pci_find_upstream_pcie_bridge(pdev);
1906 /* Secondary interface's bus number and devfn 0 */
1907 parent = pdev->bus->self;
1908 while (parent != tmp) {
1909 ret = device_context_mapped(iommu, parent->bus->number,
1913 parent = parent->bus->self;
1915 if (pci_is_pcie(tmp))
1916 return device_context_mapped(iommu, tmp->subordinate->number,
1919 return device_context_mapped(iommu, tmp->bus->number,
1923 /* Returns a number of VTD pages, but aligned to MM page size */
1924 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1927 host_addr &= ~PAGE_MASK;
1928 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1931 /* Return largest possible superpage level for a given mapping */
1932 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1933 unsigned long iov_pfn,
1934 unsigned long phy_pfn,
1935 unsigned long pages)
1937 int support, level = 1;
1938 unsigned long pfnmerge;
1940 support = domain->iommu_superpage;
1942 /* To use a large page, the virtual *and* physical addresses
1943 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1944 of them will mean we have to use smaller pages. So just
1945 merge them and check both at once. */
1946 pfnmerge = iov_pfn | phy_pfn;
1948 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1949 pages >>= VTD_STRIDE_SHIFT;
1952 pfnmerge >>= VTD_STRIDE_SHIFT;
1959 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1960 struct scatterlist *sg, unsigned long phys_pfn,
1961 unsigned long nr_pages, int prot)
1963 struct dma_pte *first_pte = NULL, *pte = NULL;
1964 phys_addr_t uninitialized_var(pteval);
1965 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1966 unsigned long sg_res;
1967 unsigned int largepage_lvl = 0;
1968 unsigned long lvl_pages = 0;
1970 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1972 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1975 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1980 sg_res = nr_pages + 1;
1981 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1984 while (nr_pages > 0) {
1988 sg_res = aligned_nrpages(sg->offset, sg->length);
1989 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1990 sg->dma_length = sg->length;
1991 pteval = page_to_phys(sg_page(sg)) | prot;
1992 phys_pfn = pteval >> VTD_PAGE_SHIFT;
1996 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1998 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2001 /* It is large page*/
2002 if (largepage_lvl > 1) {
2003 pteval |= DMA_PTE_LARGE_PAGE;
2004 /* Ensure that old small page tables are removed to make room
2005 for superpage, if they exist. */
2006 dma_pte_clear_range(domain, iov_pfn,
2007 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2008 dma_pte_free_pagetable(domain, iov_pfn,
2009 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2011 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2015 /* We don't need lock here, nobody else
2016 * touches the iova range
2018 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2020 static int dumps = 5;
2021 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2022 iov_pfn, tmp, (unsigned long long)pteval);
2025 debug_dma_dump_mappings(NULL);
2030 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2032 BUG_ON(nr_pages < lvl_pages);
2033 BUG_ON(sg_res < lvl_pages);
2035 nr_pages -= lvl_pages;
2036 iov_pfn += lvl_pages;
2037 phys_pfn += lvl_pages;
2038 pteval += lvl_pages * VTD_PAGE_SIZE;
2039 sg_res -= lvl_pages;
2041 /* If the next PTE would be the first in a new page, then we
2042 need to flush the cache on the entries we've just written.
2043 And then we'll need to recalculate 'pte', so clear it and
2044 let it get set again in the if (!pte) block above.
2046 If we're done (!nr_pages) we need to flush the cache too.
2048 Also if we've been setting superpages, we may need to
2049 recalculate 'pte' and switch back to smaller pages for the
2050 end of the mapping, if the trailing size is not enough to
2051 use another superpage (i.e. sg_res < lvl_pages). */
2053 if (!nr_pages || first_pte_in_page(pte) ||
2054 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2055 domain_flush_cache(domain, first_pte,
2056 (void *)pte - (void *)first_pte);
2060 if (!sg_res && nr_pages)
2066 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2067 struct scatterlist *sg, unsigned long nr_pages,
2070 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2073 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2074 unsigned long phys_pfn, unsigned long nr_pages,
2077 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2080 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2085 clear_context_table(iommu, bus, devfn);
2086 iommu->flush.flush_context(iommu, 0, 0, 0,
2087 DMA_CCMD_GLOBAL_INVL);
2088 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2091 static inline void unlink_domain_info(struct device_domain_info *info)
2093 assert_spin_locked(&device_domain_lock);
2094 list_del(&info->link);
2095 list_del(&info->global);
2097 info->dev->archdata.iommu = NULL;
2100 static void domain_remove_dev_info(struct dmar_domain *domain)
2102 struct device_domain_info *info;
2103 unsigned long flags, flags2;
2105 spin_lock_irqsave(&device_domain_lock, flags);
2106 while (!list_empty(&domain->devices)) {
2107 info = list_entry(domain->devices.next,
2108 struct device_domain_info, link);
2109 unlink_domain_info(info);
2110 spin_unlock_irqrestore(&device_domain_lock, flags);
2112 iommu_disable_dev_iotlb(info);
2113 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2115 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
2116 iommu_detach_dependent_devices(info->iommu, info->dev);
2117 /* clear this iommu in iommu_bmp, update iommu count
2120 spin_lock_irqsave(&domain->iommu_lock, flags2);
2121 if (test_and_clear_bit(info->iommu->seq_id,
2122 domain->iommu_bmp)) {
2123 domain->iommu_count--;
2124 domain_update_iommu_cap(domain);
2126 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2129 free_devinfo_mem(info);
2130 spin_lock_irqsave(&device_domain_lock, flags);
2132 spin_unlock_irqrestore(&device_domain_lock, flags);
2137 * Note: we use struct device->archdata.iommu stores the info
2139 static struct dmar_domain *find_domain(struct device *dev)
2141 struct device_domain_info *info;
2143 /* No lock here, assumes no domain exit in normal case */
2144 info = dev->archdata.iommu;
2146 return info->domain;
2150 static inline struct device_domain_info *
2151 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2153 struct device_domain_info *info;
2155 list_for_each_entry(info, &device_domain_list, global)
2156 if (info->iommu->segment == segment && info->bus == bus &&
2157 info->devfn == devfn)
2163 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2166 struct dmar_domain *domain)
2168 struct dmar_domain *found = NULL;
2169 struct device_domain_info *info;
2170 unsigned long flags;
2172 info = alloc_devinfo_mem();
2177 info->devfn = devfn;
2179 info->domain = domain;
2180 info->iommu = iommu;
2182 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2184 spin_lock_irqsave(&device_domain_lock, flags);
2186 found = find_domain(dev);
2188 struct device_domain_info *info2;
2189 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2191 found = info2->domain;
2194 spin_unlock_irqrestore(&device_domain_lock, flags);
2195 free_devinfo_mem(info);
2196 /* Caller must free the original domain */
2200 list_add(&info->link, &domain->devices);
2201 list_add(&info->global, &device_domain_list);
2203 dev->archdata.iommu = info;
2204 spin_unlock_irqrestore(&device_domain_lock, flags);
2209 /* domain is initialized */
2210 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2212 struct dmar_domain *domain, *free = NULL;
2213 struct intel_iommu *iommu = NULL;
2214 struct device_domain_info *info;
2215 struct pci_dev *dev_tmp = NULL;
2216 unsigned long flags;
2217 u8 bus, devfn, bridge_bus, bridge_devfn;
2219 domain = find_domain(dev);
2223 if (dev_is_pci(dev)) {
2224 struct pci_dev *pdev = to_pci_dev(dev);
2227 segment = pci_domain_nr(pdev->bus);
2228 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
2230 if (pci_is_pcie(dev_tmp)) {
2231 bridge_bus = dev_tmp->subordinate->number;
2234 bridge_bus = dev_tmp->bus->number;
2235 bridge_devfn = dev_tmp->devfn;
2237 spin_lock_irqsave(&device_domain_lock, flags);
2238 info = dmar_search_domain_by_dev_info(segment, bus, devfn);
2240 iommu = info->iommu;
2241 domain = info->domain;
2243 spin_unlock_irqrestore(&device_domain_lock, flags);
2244 /* pcie-pci bridge already has a domain, uses it */
2250 iommu = device_to_iommu(dev, &bus, &devfn);
2254 /* Allocate and initialize new domain for the device */
2255 domain = alloc_domain(false);
2258 if (iommu_attach_domain(domain, iommu)) {
2259 free_domain_mem(domain);
2264 if (domain_init(domain, gaw))
2267 /* register pcie-to-pci device */
2269 domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn,
2276 domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2284 static int iommu_identity_mapping;
2285 #define IDENTMAP_ALL 1
2286 #define IDENTMAP_GFX 2
2287 #define IDENTMAP_AZALIA 4
2289 static int iommu_domain_identity_map(struct dmar_domain *domain,
2290 unsigned long long start,
2291 unsigned long long end)
2293 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2294 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2296 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2297 dma_to_mm_pfn(last_vpfn))) {
2298 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2302 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2303 start, end, domain->id);
2305 * RMRR range might have overlap with physical memory range,
2308 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2310 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2311 last_vpfn - first_vpfn + 1,
2312 DMA_PTE_READ|DMA_PTE_WRITE);
2315 static int iommu_prepare_identity_map(struct device *dev,
2316 unsigned long long start,
2317 unsigned long long end)
2319 struct dmar_domain *domain;
2322 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2326 /* For _hardware_ passthrough, don't bother. But for software
2327 passthrough, we do it anyway -- it may indicate a memory
2328 range which is reserved in E820, so which didn't get set
2329 up to start with in si_domain */
2330 if (domain == si_domain && hw_pass_through) {
2331 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2332 dev_name(dev), start, end);
2337 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2338 dev_name(dev), start, end);
2341 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2342 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2343 dmi_get_system_info(DMI_BIOS_VENDOR),
2344 dmi_get_system_info(DMI_BIOS_VERSION),
2345 dmi_get_system_info(DMI_PRODUCT_VERSION));
2350 if (end >> agaw_to_width(domain->agaw)) {
2351 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2352 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2353 agaw_to_width(domain->agaw),
2354 dmi_get_system_info(DMI_BIOS_VENDOR),
2355 dmi_get_system_info(DMI_BIOS_VERSION),
2356 dmi_get_system_info(DMI_PRODUCT_VERSION));
2361 ret = iommu_domain_identity_map(domain, start, end);
2365 /* context entry init */
2366 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2373 domain_exit(domain);
2377 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2380 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2382 return iommu_prepare_identity_map(dev, rmrr->base_address,
2386 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2387 static inline void iommu_prepare_isa(void)
2389 struct pci_dev *pdev;
2392 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2396 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2397 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2400 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2401 "floppy might not work\n");
2405 static inline void iommu_prepare_isa(void)
2409 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2411 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2413 static int __init si_domain_init(int hw)
2415 struct dmar_drhd_unit *drhd;
2416 struct intel_iommu *iommu;
2419 si_domain = alloc_domain(false);
2423 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2425 for_each_active_iommu(iommu, drhd) {
2426 ret = iommu_attach_domain(si_domain, iommu);
2428 domain_exit(si_domain);
2433 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2434 domain_exit(si_domain);
2438 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2444 for_each_online_node(nid) {
2445 unsigned long start_pfn, end_pfn;
2448 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2449 ret = iommu_domain_identity_map(si_domain,
2450 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2459 static int identity_mapping(struct device *dev)
2461 struct device_domain_info *info;
2463 if (likely(!iommu_identity_mapping))
2466 info = dev->archdata.iommu;
2467 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2468 return (info->domain == si_domain);
2473 static int domain_add_dev_info(struct dmar_domain *domain,
2474 struct device *dev, int translation)
2476 struct dmar_domain *ndomain;
2477 struct intel_iommu *iommu;
2481 iommu = device_to_iommu(dev, &bus, &devfn);
2485 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2486 if (ndomain != domain)
2489 ret = domain_context_mapping(domain, dev, translation);
2491 domain_remove_one_dev_info(domain, dev);
2498 static bool device_has_rmrr(struct device *dev)
2500 struct dmar_rmrr_unit *rmrr;
2505 for_each_rmrr_units(rmrr) {
2507 * Return TRUE if this RMRR contains the device that
2510 for_each_active_dev_scope(rmrr->devices,
2511 rmrr->devices_cnt, i, tmp)
2521 static int iommu_should_identity_map(struct device *dev, int startup)
2524 if (dev_is_pci(dev)) {
2525 struct pci_dev *pdev = to_pci_dev(dev);
2528 * We want to prevent any device associated with an RMRR from
2529 * getting placed into the SI Domain. This is done because
2530 * problems exist when devices are moved in and out of domains
2531 * and their respective RMRR info is lost. We exempt USB devices
2532 * from this process due to their usage of RMRRs that are known
2533 * to not be needed after BIOS hand-off to OS.
2535 if (device_has_rmrr(dev) &&
2536 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2539 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2542 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2545 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2549 * We want to start off with all devices in the 1:1 domain, and
2550 * take them out later if we find they can't access all of memory.
2552 * However, we can't do this for PCI devices behind bridges,
2553 * because all PCI devices behind the same bridge will end up
2554 * with the same source-id on their transactions.
2556 * Practically speaking, we can't change things around for these
2557 * devices at run-time, because we can't be sure there'll be no
2558 * DMA transactions in flight for any of their siblings.
2560 * So PCI devices (unless they're on the root bus) as well as
2561 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2562 * the 1:1 domain, just in _case_ one of their siblings turns out
2563 * not to be able to map all of memory.
2565 if (!pci_is_pcie(pdev)) {
2566 if (!pci_is_root_bus(pdev->bus))
2568 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2570 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2573 if (device_has_rmrr(dev))
2578 * At boot time, we don't yet know if devices will be 64-bit capable.
2579 * Assume that they will — if they turn out not to be, then we can
2580 * take them out of the 1:1 domain later.
2584 * If the device's dma_mask is less than the system's memory
2585 * size then this is not a candidate for identity mapping.
2587 u64 dma_mask = *dev->dma_mask;
2589 if (dev->coherent_dma_mask &&
2590 dev->coherent_dma_mask < dma_mask)
2591 dma_mask = dev->coherent_dma_mask;
2593 return dma_mask >= dma_get_required_mask(dev);
2599 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2603 if (!iommu_should_identity_map(dev, 1))
2606 ret = domain_add_dev_info(si_domain, dev,
2607 hw ? CONTEXT_TT_PASS_THROUGH :
2608 CONTEXT_TT_MULTI_LEVEL);
2610 pr_info("IOMMU: %s identity mapping for device %s\n",
2611 hw ? "hardware" : "software", dev_name(dev));
2612 else if (ret == -ENODEV)
2613 /* device not associated with an iommu */
2620 static int __init iommu_prepare_static_identity_mapping(int hw)
2622 struct pci_dev *pdev = NULL;
2623 struct dmar_drhd_unit *drhd;
2624 struct intel_iommu *iommu;
2629 ret = si_domain_init(hw);
2633 for_each_pci_dev(pdev) {
2634 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2639 for_each_active_iommu(iommu, drhd)
2640 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2641 struct acpi_device_physical_node *pn;
2642 struct acpi_device *adev;
2644 if (dev->bus != &acpi_bus_type)
2647 adev= to_acpi_device(dev);
2648 mutex_lock(&adev->physical_node_lock);
2649 list_for_each_entry(pn, &adev->physical_node_list, node) {
2650 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2654 mutex_unlock(&adev->physical_node_lock);
2662 static int __init init_dmars(void)
2664 struct dmar_drhd_unit *drhd;
2665 struct dmar_rmrr_unit *rmrr;
2667 struct intel_iommu *iommu;
2673 * initialize and program root entry to not present
2676 for_each_drhd_unit(drhd) {
2678 * lock not needed as this is only incremented in the single
2679 * threaded kernel __init code path all other access are read
2682 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2686 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2687 IOMMU_UNITS_SUPPORTED);
2690 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2693 printk(KERN_ERR "Allocating global iommu array failed\n");
2698 deferred_flush = kzalloc(g_num_of_iommus *
2699 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2700 if (!deferred_flush) {
2705 for_each_active_iommu(iommu, drhd) {
2706 g_iommus[iommu->seq_id] = iommu;
2708 ret = iommu_init_domains(iommu);
2714 * we could share the same root & context tables
2715 * among all IOMMU's. Need to Split it later.
2717 ret = iommu_alloc_root_entry(iommu);
2719 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2722 if (!ecap_pass_through(iommu->ecap))
2723 hw_pass_through = 0;
2727 * Start from the sane iommu hardware state.
2729 for_each_active_iommu(iommu, drhd) {
2731 * If the queued invalidation is already initialized by us
2732 * (for example, while enabling interrupt-remapping) then
2733 * we got the things already rolling from a sane state.
2739 * Clear any previous faults.
2741 dmar_fault(-1, iommu);
2743 * Disable queued invalidation if supported and already enabled
2744 * before OS handover.
2746 dmar_disable_qi(iommu);
2749 for_each_active_iommu(iommu, drhd) {
2750 if (dmar_enable_qi(iommu)) {
2752 * Queued Invalidate not enabled, use Register Based
2755 iommu->flush.flush_context = __iommu_flush_context;
2756 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2757 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2760 (unsigned long long)drhd->reg_base_addr);
2762 iommu->flush.flush_context = qi_flush_context;
2763 iommu->flush.flush_iotlb = qi_flush_iotlb;
2764 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2767 (unsigned long long)drhd->reg_base_addr);
2771 if (iommu_pass_through)
2772 iommu_identity_mapping |= IDENTMAP_ALL;
2774 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2775 iommu_identity_mapping |= IDENTMAP_GFX;
2778 check_tylersburg_isoch();
2781 * If pass through is not set or not enabled, setup context entries for
2782 * identity mappings for rmrr, gfx, and isa and may fall back to static
2783 * identity mapping if iommu_identity_mapping is set.
2785 if (iommu_identity_mapping) {
2786 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2788 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2794 * for each dev attached to rmrr
2796 * locate drhd for dev, alloc domain for dev
2797 * allocate free domain
2798 * allocate page table entries for rmrr
2799 * if context not allocated for bus
2800 * allocate and init context
2801 * set present in root table for this bus
2802 * init context with domain, translation etc
2806 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2807 for_each_rmrr_units(rmrr) {
2808 /* some BIOS lists non-exist devices in DMAR table. */
2809 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2811 ret = iommu_prepare_rmrr_dev(rmrr, dev);
2814 "IOMMU: mapping reserved region failed\n");
2818 iommu_prepare_isa();
2823 * global invalidate context cache
2824 * global invalidate iotlb
2825 * enable translation
2827 for_each_iommu(iommu, drhd) {
2828 if (drhd->ignored) {
2830 * we always have to disable PMRs or DMA may fail on
2834 iommu_disable_protect_mem_regions(iommu);
2838 iommu_flush_write_buffer(iommu);
2840 ret = dmar_set_interrupt(iommu);
2844 iommu_set_root_entry(iommu);
2846 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2847 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2849 ret = iommu_enable_translation(iommu);
2853 iommu_disable_protect_mem_regions(iommu);
2859 for_each_active_iommu(iommu, drhd)
2860 free_dmar_iommu(iommu);
2861 kfree(deferred_flush);
2868 /* This takes a number of _MM_ pages, not VTD pages */
2869 static struct iova *intel_alloc_iova(struct device *dev,
2870 struct dmar_domain *domain,
2871 unsigned long nrpages, uint64_t dma_mask)
2873 struct iova *iova = NULL;
2875 /* Restrict dma_mask to the width that the iommu can handle */
2876 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2878 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2880 * First try to allocate an io virtual address in
2881 * DMA_BIT_MASK(32) and if that fails then try allocating
2884 iova = alloc_iova(&domain->iovad, nrpages,
2885 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2889 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2890 if (unlikely(!iova)) {
2891 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2892 nrpages, dev_name(dev));
2899 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2901 struct dmar_domain *domain;
2904 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2906 printk(KERN_ERR "Allocating domain for %s failed",
2911 /* make sure context mapping is ok */
2912 if (unlikely(!domain_context_mapped(dev))) {
2913 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2915 printk(KERN_ERR "Domain context map for %s failed",
2924 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2926 struct device_domain_info *info;
2928 /* No lock here, assumes no domain exit in normal case */
2929 info = dev->archdata.iommu;
2931 return info->domain;
2933 return __get_valid_domain_for_dev(dev);
2936 static int iommu_dummy(struct device *dev)
2938 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2941 /* Check if the dev needs to go through non-identity map and unmap process.*/
2942 static int iommu_no_mapping(struct device *dev)
2946 if (iommu_dummy(dev))
2949 if (!iommu_identity_mapping)
2952 found = identity_mapping(dev);
2954 if (iommu_should_identity_map(dev, 0))
2958 * 32 bit DMA is removed from si_domain and fall back
2959 * to non-identity mapping.
2961 domain_remove_one_dev_info(si_domain, dev);
2962 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2968 * In case of a detached 64 bit DMA device from vm, the device
2969 * is put into si_domain for identity mapping.
2971 if (iommu_should_identity_map(dev, 0)) {
2973 ret = domain_add_dev_info(si_domain, dev,
2975 CONTEXT_TT_PASS_THROUGH :
2976 CONTEXT_TT_MULTI_LEVEL);
2978 printk(KERN_INFO "64bit %s uses identity mapping\n",
2988 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
2989 size_t size, int dir, u64 dma_mask)
2991 struct dmar_domain *domain;
2992 phys_addr_t start_paddr;
2996 struct intel_iommu *iommu;
2997 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2999 BUG_ON(dir == DMA_NONE);
3001 if (iommu_no_mapping(dev))
3004 domain = get_valid_domain_for_dev(dev);
3008 iommu = domain_get_iommu(domain);
3009 size = aligned_nrpages(paddr, size);
3011 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3016 * Check if DMAR supports zero-length reads on write only
3019 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3020 !cap_zlr(iommu->cap))
3021 prot |= DMA_PTE_READ;
3022 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3023 prot |= DMA_PTE_WRITE;
3025 * paddr - (paddr + size) might be partial page, we should map the whole
3026 * page. Note: if two part of one page are separately mapped, we
3027 * might have two guest_addr mapping to the same host paddr, but this
3028 * is not a big problem
3030 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3031 mm_to_dma_pfn(paddr_pfn), size, prot);
3035 /* it's a non-present to present mapping. Only flush if caching mode */
3036 if (cap_caching_mode(iommu->cap))
3037 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3039 iommu_flush_write_buffer(iommu);
3041 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3042 start_paddr += paddr & ~PAGE_MASK;
3047 __free_iova(&domain->iovad, iova);
3048 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
3049 dev_name(dev), size, (unsigned long long)paddr, dir);
3053 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3054 unsigned long offset, size_t size,
3055 enum dma_data_direction dir,
3056 struct dma_attrs *attrs)
3058 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3059 dir, *dev->dma_mask);
3062 static void flush_unmaps(void)
3068 /* just flush them all */
3069 for (i = 0; i < g_num_of_iommus; i++) {
3070 struct intel_iommu *iommu = g_iommus[i];
3074 if (!deferred_flush[i].next)
3077 /* In caching mode, global flushes turn emulation expensive */
3078 if (!cap_caching_mode(iommu->cap))
3079 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3080 DMA_TLB_GLOBAL_FLUSH);
3081 for (j = 0; j < deferred_flush[i].next; j++) {
3083 struct iova *iova = deferred_flush[i].iova[j];
3084 struct dmar_domain *domain = deferred_flush[i].domain[j];
3086 /* On real hardware multiple invalidations are expensive */
3087 if (cap_caching_mode(iommu->cap))
3088 iommu_flush_iotlb_psi(iommu, domain->id,
3089 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3090 !deferred_flush[i].freelist[j], 0);
3092 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3093 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3094 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3096 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3097 if (deferred_flush[i].freelist[j])
3098 dma_free_pagelist(deferred_flush[i].freelist[j]);
3100 deferred_flush[i].next = 0;
3106 static void flush_unmaps_timeout(unsigned long data)
3108 unsigned long flags;
3110 spin_lock_irqsave(&async_umap_flush_lock, flags);
3112 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3115 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3117 unsigned long flags;
3119 struct intel_iommu *iommu;
3121 spin_lock_irqsave(&async_umap_flush_lock, flags);
3122 if (list_size == HIGH_WATER_MARK)
3125 iommu = domain_get_iommu(dom);
3126 iommu_id = iommu->seq_id;
3128 next = deferred_flush[iommu_id].next;
3129 deferred_flush[iommu_id].domain[next] = dom;
3130 deferred_flush[iommu_id].iova[next] = iova;
3131 deferred_flush[iommu_id].freelist[next] = freelist;
3132 deferred_flush[iommu_id].next++;
3135 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3139 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3142 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3143 size_t size, enum dma_data_direction dir,
3144 struct dma_attrs *attrs)
3146 struct dmar_domain *domain;
3147 unsigned long start_pfn, last_pfn;
3149 struct intel_iommu *iommu;
3150 struct page *freelist;
3152 if (iommu_no_mapping(dev))
3155 domain = find_domain(dev);
3158 iommu = domain_get_iommu(domain);
3160 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3161 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3162 (unsigned long long)dev_addr))
3165 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3166 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3168 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3169 dev_name(dev), start_pfn, last_pfn);
3171 freelist = domain_unmap(domain, start_pfn, last_pfn);
3173 if (intel_iommu_strict) {
3174 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3175 last_pfn - start_pfn + 1, !freelist, 0);
3177 __free_iova(&domain->iovad, iova);
3178 dma_free_pagelist(freelist);
3180 add_unmap(domain, iova, freelist);
3182 * queue up the release of the unmap to save the 1/6th of the
3183 * cpu used up by the iotlb flush operation...
3188 static void *intel_alloc_coherent(struct device *dev, size_t size,
3189 dma_addr_t *dma_handle, gfp_t flags,
3190 struct dma_attrs *attrs)
3195 size = PAGE_ALIGN(size);
3196 order = get_order(size);
3198 if (!iommu_no_mapping(dev))
3199 flags &= ~(GFP_DMA | GFP_DMA32);
3200 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3201 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3207 vaddr = (void *)__get_free_pages(flags, order);
3210 memset(vaddr, 0, size);
3212 *dma_handle = __intel_map_single(dev, virt_to_bus(vaddr), size,
3214 dev->coherent_dma_mask);
3217 free_pages((unsigned long)vaddr, order);
3221 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3222 dma_addr_t dma_handle, struct dma_attrs *attrs)
3226 size = PAGE_ALIGN(size);
3227 order = get_order(size);
3229 intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
3230 free_pages((unsigned long)vaddr, order);
3233 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3234 int nelems, enum dma_data_direction dir,
3235 struct dma_attrs *attrs)
3237 struct dmar_domain *domain;
3238 unsigned long start_pfn, last_pfn;
3240 struct intel_iommu *iommu;
3241 struct page *freelist;
3243 if (iommu_no_mapping(dev))
3246 domain = find_domain(dev);
3249 iommu = domain_get_iommu(domain);
3251 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3252 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3253 (unsigned long long)sglist[0].dma_address))
3256 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3257 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3259 freelist = domain_unmap(domain, start_pfn, last_pfn);
3261 if (intel_iommu_strict) {
3262 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3263 last_pfn - start_pfn + 1, !freelist, 0);
3265 __free_iova(&domain->iovad, iova);
3266 dma_free_pagelist(freelist);
3268 add_unmap(domain, iova, freelist);
3270 * queue up the release of the unmap to save the 1/6th of the
3271 * cpu used up by the iotlb flush operation...
3276 static int intel_nontranslate_map_sg(struct device *hddev,
3277 struct scatterlist *sglist, int nelems, int dir)
3280 struct scatterlist *sg;
3282 for_each_sg(sglist, sg, nelems, i) {
3283 BUG_ON(!sg_page(sg));
3284 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3285 sg->dma_length = sg->length;
3290 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3291 enum dma_data_direction dir, struct dma_attrs *attrs)
3294 struct dmar_domain *domain;
3297 struct iova *iova = NULL;
3299 struct scatterlist *sg;
3300 unsigned long start_vpfn;
3301 struct intel_iommu *iommu;
3303 BUG_ON(dir == DMA_NONE);
3304 if (iommu_no_mapping(dev))
3305 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3307 domain = get_valid_domain_for_dev(dev);
3311 iommu = domain_get_iommu(domain);
3313 for_each_sg(sglist, sg, nelems, i)
3314 size += aligned_nrpages(sg->offset, sg->length);
3316 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3319 sglist->dma_length = 0;
3324 * Check if DMAR supports zero-length reads on write only
3327 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3328 !cap_zlr(iommu->cap))
3329 prot |= DMA_PTE_READ;
3330 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3331 prot |= DMA_PTE_WRITE;
3333 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3335 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3336 if (unlikely(ret)) {
3337 /* clear the page */
3338 dma_pte_clear_range(domain, start_vpfn,
3339 start_vpfn + size - 1);
3340 /* free page tables */
3341 dma_pte_free_pagetable(domain, start_vpfn,
3342 start_vpfn + size - 1);
3344 __free_iova(&domain->iovad, iova);
3348 /* it's a non-present to present mapping. Only flush if caching mode */
3349 if (cap_caching_mode(iommu->cap))
3350 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3352 iommu_flush_write_buffer(iommu);
3357 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3362 struct dma_map_ops intel_dma_ops = {
3363 .alloc = intel_alloc_coherent,
3364 .free = intel_free_coherent,
3365 .map_sg = intel_map_sg,
3366 .unmap_sg = intel_unmap_sg,
3367 .map_page = intel_map_page,
3368 .unmap_page = intel_unmap_page,
3369 .mapping_error = intel_mapping_error,
3372 static inline int iommu_domain_cache_init(void)
3376 iommu_domain_cache = kmem_cache_create("iommu_domain",
3377 sizeof(struct dmar_domain),
3382 if (!iommu_domain_cache) {
3383 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3390 static inline int iommu_devinfo_cache_init(void)
3394 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3395 sizeof(struct device_domain_info),
3399 if (!iommu_devinfo_cache) {
3400 printk(KERN_ERR "Couldn't create devinfo cache\n");
3407 static inline int iommu_iova_cache_init(void)
3411 iommu_iova_cache = kmem_cache_create("iommu_iova",
3412 sizeof(struct iova),
3416 if (!iommu_iova_cache) {
3417 printk(KERN_ERR "Couldn't create iova cache\n");
3424 static int __init iommu_init_mempool(void)
3427 ret = iommu_iova_cache_init();
3431 ret = iommu_domain_cache_init();
3435 ret = iommu_devinfo_cache_init();
3439 kmem_cache_destroy(iommu_domain_cache);
3441 kmem_cache_destroy(iommu_iova_cache);
3446 static void __init iommu_exit_mempool(void)
3448 kmem_cache_destroy(iommu_devinfo_cache);
3449 kmem_cache_destroy(iommu_domain_cache);
3450 kmem_cache_destroy(iommu_iova_cache);
3454 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3456 struct dmar_drhd_unit *drhd;
3460 /* We know that this device on this chipset has its own IOMMU.
3461 * If we find it under a different IOMMU, then the BIOS is lying
3462 * to us. Hope that the IOMMU for this device is actually
3463 * disabled, and it needs no translation...
3465 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3467 /* "can't" happen */
3468 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3471 vtbar &= 0xffff0000;
3473 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3474 drhd = dmar_find_matched_drhd_unit(pdev);
3475 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3476 TAINT_FIRMWARE_WORKAROUND,
3477 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3478 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3480 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3482 static void __init init_no_remapping_devices(void)
3484 struct dmar_drhd_unit *drhd;
3488 for_each_drhd_unit(drhd) {
3489 if (!drhd->include_all) {
3490 for_each_active_dev_scope(drhd->devices,
3491 drhd->devices_cnt, i, dev)
3493 /* ignore DMAR unit if no devices exist */
3494 if (i == drhd->devices_cnt)
3499 for_each_active_drhd_unit(drhd) {
3500 if (drhd->include_all)
3503 for_each_active_dev_scope(drhd->devices,
3504 drhd->devices_cnt, i, dev)
3505 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3507 if (i < drhd->devices_cnt)
3510 /* This IOMMU has *only* gfx devices. Either bypass it or
3511 set the gfx_mapped flag, as appropriate */
3513 intel_iommu_gfx_mapped = 1;
3516 for_each_active_dev_scope(drhd->devices,
3517 drhd->devices_cnt, i, dev)
3518 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3523 #ifdef CONFIG_SUSPEND
3524 static int init_iommu_hw(void)
3526 struct dmar_drhd_unit *drhd;
3527 struct intel_iommu *iommu = NULL;
3529 for_each_active_iommu(iommu, drhd)
3531 dmar_reenable_qi(iommu);
3533 for_each_iommu(iommu, drhd) {
3534 if (drhd->ignored) {
3536 * we always have to disable PMRs or DMA may fail on
3540 iommu_disable_protect_mem_regions(iommu);
3544 iommu_flush_write_buffer(iommu);
3546 iommu_set_root_entry(iommu);
3548 iommu->flush.flush_context(iommu, 0, 0, 0,
3549 DMA_CCMD_GLOBAL_INVL);
3550 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3551 DMA_TLB_GLOBAL_FLUSH);
3552 if (iommu_enable_translation(iommu))
3554 iommu_disable_protect_mem_regions(iommu);
3560 static void iommu_flush_all(void)
3562 struct dmar_drhd_unit *drhd;
3563 struct intel_iommu *iommu;
3565 for_each_active_iommu(iommu, drhd) {
3566 iommu->flush.flush_context(iommu, 0, 0, 0,
3567 DMA_CCMD_GLOBAL_INVL);
3568 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3569 DMA_TLB_GLOBAL_FLUSH);
3573 static int iommu_suspend(void)
3575 struct dmar_drhd_unit *drhd;
3576 struct intel_iommu *iommu = NULL;
3579 for_each_active_iommu(iommu, drhd) {
3580 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3582 if (!iommu->iommu_state)
3588 for_each_active_iommu(iommu, drhd) {
3589 iommu_disable_translation(iommu);
3591 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3593 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3594 readl(iommu->reg + DMAR_FECTL_REG);
3595 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3596 readl(iommu->reg + DMAR_FEDATA_REG);
3597 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3598 readl(iommu->reg + DMAR_FEADDR_REG);
3599 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3600 readl(iommu->reg + DMAR_FEUADDR_REG);
3602 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3607 for_each_active_iommu(iommu, drhd)
3608 kfree(iommu->iommu_state);
3613 static void iommu_resume(void)
3615 struct dmar_drhd_unit *drhd;
3616 struct intel_iommu *iommu = NULL;
3619 if (init_iommu_hw()) {
3621 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3623 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3627 for_each_active_iommu(iommu, drhd) {
3629 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3631 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3632 iommu->reg + DMAR_FECTL_REG);
3633 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3634 iommu->reg + DMAR_FEDATA_REG);
3635 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3636 iommu->reg + DMAR_FEADDR_REG);
3637 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3638 iommu->reg + DMAR_FEUADDR_REG);
3640 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3643 for_each_active_iommu(iommu, drhd)
3644 kfree(iommu->iommu_state);
3647 static struct syscore_ops iommu_syscore_ops = {
3648 .resume = iommu_resume,
3649 .suspend = iommu_suspend,
3652 static void __init init_iommu_pm_ops(void)
3654 register_syscore_ops(&iommu_syscore_ops);
3658 static inline void init_iommu_pm_ops(void) {}
3659 #endif /* CONFIG_PM */
3662 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3664 struct acpi_dmar_reserved_memory *rmrr;
3665 struct dmar_rmrr_unit *rmrru;
3667 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3671 rmrru->hdr = header;
3672 rmrr = (struct acpi_dmar_reserved_memory *)header;
3673 rmrru->base_address = rmrr->base_address;
3674 rmrru->end_address = rmrr->end_address;
3675 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3676 ((void *)rmrr) + rmrr->header.length,
3677 &rmrru->devices_cnt);
3678 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3683 list_add(&rmrru->list, &dmar_rmrr_units);
3688 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3690 struct acpi_dmar_atsr *atsr;
3691 struct dmar_atsr_unit *atsru;
3693 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3694 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3699 atsru->include_all = atsr->flags & 0x1;
3700 if (!atsru->include_all) {
3701 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3702 (void *)atsr + atsr->header.length,
3703 &atsru->devices_cnt);
3704 if (atsru->devices_cnt && atsru->devices == NULL) {
3710 list_add_rcu(&atsru->list, &dmar_atsr_units);
3715 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3717 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3721 static void intel_iommu_free_dmars(void)
3723 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3724 struct dmar_atsr_unit *atsru, *atsr_n;
3726 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3727 list_del(&rmrru->list);
3728 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3732 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3733 list_del(&atsru->list);
3734 intel_iommu_free_atsr(atsru);
3738 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3741 struct pci_bus *bus;
3742 struct pci_dev *bridge = NULL;
3744 struct acpi_dmar_atsr *atsr;
3745 struct dmar_atsr_unit *atsru;
3747 dev = pci_physfn(dev);
3748 for (bus = dev->bus; bus; bus = bus->parent) {
3750 if (!bridge || !pci_is_pcie(bridge) ||
3751 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3753 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3760 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3761 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3762 if (atsr->segment != pci_domain_nr(dev->bus))
3765 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3766 if (tmp == &bridge->dev)
3769 if (atsru->include_all)
3779 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3782 struct dmar_rmrr_unit *rmrru;
3783 struct dmar_atsr_unit *atsru;
3784 struct acpi_dmar_atsr *atsr;
3785 struct acpi_dmar_reserved_memory *rmrr;
3787 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3790 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3791 rmrr = container_of(rmrru->hdr,
3792 struct acpi_dmar_reserved_memory, header);
3793 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3794 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3795 ((void *)rmrr) + rmrr->header.length,
3796 rmrr->segment, rmrru->devices,
3797 rmrru->devices_cnt);
3802 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3803 if (dmar_remove_dev_scope(info, rmrr->segment,
3804 rmrru->devices, rmrru->devices_cnt))
3809 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3810 if (atsru->include_all)
3813 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3814 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3815 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3816 (void *)atsr + atsr->header.length,
3817 atsr->segment, atsru->devices,
3818 atsru->devices_cnt);
3823 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3824 if (dmar_remove_dev_scope(info, atsr->segment,
3825 atsru->devices, atsru->devices_cnt))
3834 * Here we only respond to action of unbound device from driver.
3836 * Added device is not attached to its DMAR domain here yet. That will happen
3837 * when mapping the device to iova.
3839 static int device_notifier(struct notifier_block *nb,
3840 unsigned long action, void *data)
3842 struct device *dev = data;
3843 struct dmar_domain *domain;
3845 if (iommu_dummy(dev))
3848 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3849 action != BUS_NOTIFY_DEL_DEVICE)
3852 domain = find_domain(dev);
3856 down_read(&dmar_global_lock);
3857 domain_remove_one_dev_info(domain, dev);
3858 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3859 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3860 list_empty(&domain->devices))
3861 domain_exit(domain);
3862 up_read(&dmar_global_lock);
3867 static struct notifier_block device_nb = {
3868 .notifier_call = device_notifier,
3871 static int intel_iommu_memory_notifier(struct notifier_block *nb,
3872 unsigned long val, void *v)
3874 struct memory_notify *mhp = v;
3875 unsigned long long start, end;
3876 unsigned long start_vpfn, last_vpfn;
3879 case MEM_GOING_ONLINE:
3880 start = mhp->start_pfn << PAGE_SHIFT;
3881 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3882 if (iommu_domain_identity_map(si_domain, start, end)) {
3883 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3890 case MEM_CANCEL_ONLINE:
3891 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3892 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3893 while (start_vpfn <= last_vpfn) {
3895 struct dmar_drhd_unit *drhd;
3896 struct intel_iommu *iommu;
3897 struct page *freelist;
3899 iova = find_iova(&si_domain->iovad, start_vpfn);
3901 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3906 iova = split_and_remove_iova(&si_domain->iovad, iova,
3907 start_vpfn, last_vpfn);
3909 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3910 start_vpfn, last_vpfn);
3914 freelist = domain_unmap(si_domain, iova->pfn_lo,
3918 for_each_active_iommu(iommu, drhd)
3919 iommu_flush_iotlb_psi(iommu, si_domain->id,
3921 iova->pfn_hi - iova->pfn_lo + 1,
3924 dma_free_pagelist(freelist);
3926 start_vpfn = iova->pfn_hi + 1;
3927 free_iova_mem(iova);
3935 static struct notifier_block intel_iommu_memory_nb = {
3936 .notifier_call = intel_iommu_memory_notifier,
3940 int __init intel_iommu_init(void)
3943 struct dmar_drhd_unit *drhd;
3944 struct intel_iommu *iommu;
3946 /* VT-d is required for a TXT/tboot launch, so enforce that */
3947 force_on = tboot_force_iommu();
3949 if (iommu_init_mempool()) {
3951 panic("tboot: Failed to initialize iommu memory\n");
3955 down_write(&dmar_global_lock);
3956 if (dmar_table_init()) {
3958 panic("tboot: Failed to initialize DMAR table\n");
3963 * Disable translation if already enabled prior to OS handover.
3965 for_each_active_iommu(iommu, drhd)
3966 if (iommu->gcmd & DMA_GCMD_TE)
3967 iommu_disable_translation(iommu);
3969 if (dmar_dev_scope_init() < 0) {
3971 panic("tboot: Failed to initialize DMAR device scope\n");
3975 if (no_iommu || dmar_disabled)
3978 if (list_empty(&dmar_rmrr_units))
3979 printk(KERN_INFO "DMAR: No RMRR found\n");
3981 if (list_empty(&dmar_atsr_units))
3982 printk(KERN_INFO "DMAR: No ATSR found\n");
3984 if (dmar_init_reserved_ranges()) {
3986 panic("tboot: Failed to reserve iommu ranges\n");
3987 goto out_free_reserved_range;
3990 init_no_remapping_devices();
3995 panic("tboot: Failed to initialize DMARs\n");
3996 printk(KERN_ERR "IOMMU: dmar init failed\n");
3997 goto out_free_reserved_range;
3999 up_write(&dmar_global_lock);
4001 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4003 init_timer(&unmap_timer);
4004 #ifdef CONFIG_SWIOTLB
4007 dma_ops = &intel_dma_ops;
4009 init_iommu_pm_ops();
4011 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4012 bus_register_notifier(&pci_bus_type, &device_nb);
4013 if (si_domain && !hw_pass_through)
4014 register_memory_notifier(&intel_iommu_memory_nb);
4016 intel_iommu_enabled = 1;
4020 out_free_reserved_range:
4021 put_iova_domain(&reserved_iova_list);
4023 intel_iommu_free_dmars();
4024 up_write(&dmar_global_lock);
4025 iommu_exit_mempool();
4029 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4032 struct pci_dev *tmp, *parent, *pdev;
4034 if (!iommu || !dev || !dev_is_pci(dev))
4037 pdev = to_pci_dev(dev);
4039 /* dependent device detach */
4040 tmp = pci_find_upstream_pcie_bridge(pdev);
4041 /* Secondary interface's bus number and devfn 0 */
4043 parent = pdev->bus->self;
4044 while (parent != tmp) {
4045 iommu_detach_dev(iommu, parent->bus->number,
4047 parent = parent->bus->self;
4049 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
4050 iommu_detach_dev(iommu,
4051 tmp->subordinate->number, 0);
4052 else /* this is a legacy PCI bridge */
4053 iommu_detach_dev(iommu, tmp->bus->number,
4058 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4061 struct device_domain_info *info, *tmp;
4062 struct intel_iommu *iommu;
4063 unsigned long flags;
4067 iommu = device_to_iommu(dev, &bus, &devfn);
4071 spin_lock_irqsave(&device_domain_lock, flags);
4072 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4073 if (info->iommu == iommu && info->bus == bus &&
4074 info->devfn == devfn) {
4075 unlink_domain_info(info);
4076 spin_unlock_irqrestore(&device_domain_lock, flags);
4078 iommu_disable_dev_iotlb(info);
4079 iommu_detach_dev(iommu, info->bus, info->devfn);
4080 iommu_detach_dependent_devices(iommu, dev);
4081 free_devinfo_mem(info);
4083 spin_lock_irqsave(&device_domain_lock, flags);
4091 /* if there is no other devices under the same iommu
4092 * owned by this domain, clear this iommu in iommu_bmp
4093 * update iommu count and coherency
4095 if (info->iommu == iommu)
4099 spin_unlock_irqrestore(&device_domain_lock, flags);
4102 unsigned long tmp_flags;
4103 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
4104 clear_bit(iommu->seq_id, domain->iommu_bmp);
4105 domain->iommu_count--;
4106 domain_update_iommu_cap(domain);
4107 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
4109 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4110 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4111 spin_lock_irqsave(&iommu->lock, tmp_flags);
4112 clear_bit(domain->id, iommu->domain_ids);
4113 iommu->domains[domain->id] = NULL;
4114 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4119 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4123 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
4124 domain_reserve_special_ranges(domain);
4126 /* calculate AGAW */
4127 domain->gaw = guest_width;
4128 adjust_width = guestwidth_to_adjustwidth(guest_width);
4129 domain->agaw = width_to_agaw(adjust_width);
4131 domain->iommu_coherency = 0;
4132 domain->iommu_snooping = 0;
4133 domain->iommu_superpage = 0;
4134 domain->max_addr = 0;
4137 /* always allocate the top pgd */
4138 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4141 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4145 static int intel_iommu_domain_init(struct iommu_domain *domain)
4147 struct dmar_domain *dmar_domain;
4149 dmar_domain = alloc_domain(true);
4152 "intel_iommu_domain_init: dmar_domain == NULL\n");
4155 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4157 "intel_iommu_domain_init() failed\n");
4158 domain_exit(dmar_domain);
4161 domain_update_iommu_cap(dmar_domain);
4162 domain->priv = dmar_domain;
4164 domain->geometry.aperture_start = 0;
4165 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4166 domain->geometry.force_aperture = true;
4171 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
4173 struct dmar_domain *dmar_domain = domain->priv;
4175 domain->priv = NULL;
4176 domain_exit(dmar_domain);
4179 static int intel_iommu_attach_device(struct iommu_domain *domain,
4182 struct dmar_domain *dmar_domain = domain->priv;
4183 struct intel_iommu *iommu;
4187 /* normally dev is not mapped */
4188 if (unlikely(domain_context_mapped(dev))) {
4189 struct dmar_domain *old_domain;
4191 old_domain = find_domain(dev);
4193 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4194 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4195 domain_remove_one_dev_info(old_domain, dev);
4197 domain_remove_dev_info(old_domain);
4201 iommu = device_to_iommu(dev, &bus, &devfn);
4205 /* check if this iommu agaw is sufficient for max mapped address */
4206 addr_width = agaw_to_width(iommu->agaw);
4207 if (addr_width > cap_mgaw(iommu->cap))
4208 addr_width = cap_mgaw(iommu->cap);
4210 if (dmar_domain->max_addr > (1LL << addr_width)) {
4211 printk(KERN_ERR "%s: iommu width (%d) is not "
4212 "sufficient for the mapped address (%llx)\n",
4213 __func__, addr_width, dmar_domain->max_addr);
4216 dmar_domain->gaw = addr_width;
4219 * Knock out extra levels of page tables if necessary
4221 while (iommu->agaw < dmar_domain->agaw) {
4222 struct dma_pte *pte;
4224 pte = dmar_domain->pgd;
4225 if (dma_pte_present(pte)) {
4226 dmar_domain->pgd = (struct dma_pte *)
4227 phys_to_virt(dma_pte_addr(pte));
4228 free_pgtable_page(pte);
4230 dmar_domain->agaw--;
4233 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4236 static void intel_iommu_detach_device(struct iommu_domain *domain,
4239 struct dmar_domain *dmar_domain = domain->priv;
4241 domain_remove_one_dev_info(dmar_domain, dev);
4244 static int intel_iommu_map(struct iommu_domain *domain,
4245 unsigned long iova, phys_addr_t hpa,
4246 size_t size, int iommu_prot)
4248 struct dmar_domain *dmar_domain = domain->priv;
4253 if (iommu_prot & IOMMU_READ)
4254 prot |= DMA_PTE_READ;
4255 if (iommu_prot & IOMMU_WRITE)
4256 prot |= DMA_PTE_WRITE;
4257 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4258 prot |= DMA_PTE_SNP;
4260 max_addr = iova + size;
4261 if (dmar_domain->max_addr < max_addr) {
4264 /* check if minimum agaw is sufficient for mapped address */
4265 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4266 if (end < max_addr) {
4267 printk(KERN_ERR "%s: iommu width (%d) is not "
4268 "sufficient for the mapped address (%llx)\n",
4269 __func__, dmar_domain->gaw, max_addr);
4272 dmar_domain->max_addr = max_addr;
4274 /* Round up size to next multiple of PAGE_SIZE, if it and
4275 the low bits of hpa would take us onto the next page */
4276 size = aligned_nrpages(hpa, size);
4277 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4278 hpa >> VTD_PAGE_SHIFT, size, prot);
4282 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4283 unsigned long iova, size_t size)
4285 struct dmar_domain *dmar_domain = domain->priv;
4286 struct page *freelist = NULL;
4287 struct intel_iommu *iommu;
4288 unsigned long start_pfn, last_pfn;
4289 unsigned int npages;
4290 int iommu_id, num, ndomains, level = 0;
4292 /* Cope with horrid API which requires us to unmap more than the
4293 size argument if it happens to be a large-page mapping. */
4294 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4297 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4298 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4300 start_pfn = iova >> VTD_PAGE_SHIFT;
4301 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4303 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4305 npages = last_pfn - start_pfn + 1;
4307 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4308 iommu = g_iommus[iommu_id];
4311 * find bit position of dmar_domain
4313 ndomains = cap_ndoms(iommu->cap);
4314 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4315 if (iommu->domains[num] == dmar_domain)
4316 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4317 npages, !freelist, 0);
4322 dma_free_pagelist(freelist);
4324 if (dmar_domain->max_addr == iova + size)
4325 dmar_domain->max_addr = iova;
4330 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4333 struct dmar_domain *dmar_domain = domain->priv;
4334 struct dma_pte *pte;
4338 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4340 phys = dma_pte_addr(pte);
4345 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4348 struct dmar_domain *dmar_domain = domain->priv;
4350 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4351 return dmar_domain->iommu_snooping;
4352 if (cap == IOMMU_CAP_INTR_REMAP)
4353 return irq_remapping_enabled;
4358 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4360 static int intel_iommu_add_device(struct device *dev)
4362 struct pci_dev *pdev = to_pci_dev(dev);
4363 struct pci_dev *bridge, *dma_pdev = NULL;
4364 struct iommu_group *group;
4368 if (!device_to_iommu(dev, &bus, &devfn))
4371 bridge = pci_find_upstream_pcie_bridge(pdev);
4373 if (pci_is_pcie(bridge))
4374 dma_pdev = pci_get_domain_bus_and_slot(
4375 pci_domain_nr(pdev->bus),
4376 bridge->subordinate->number, 0);
4378 dma_pdev = pci_dev_get(bridge);
4380 dma_pdev = pci_dev_get(pdev);
4382 /* Account for quirked devices */
4383 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4386 * If it's a multifunction device that does not support our
4387 * required ACS flags, add to the same group as lowest numbered
4388 * function that also does not suport the required ACS flags.
4390 if (dma_pdev->multifunction &&
4391 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4392 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4394 for (i = 0; i < 8; i++) {
4395 struct pci_dev *tmp;
4397 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4401 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4402 swap_pci_ref(&dma_pdev, tmp);
4410 * Devices on the root bus go through the iommu. If that's not us,
4411 * find the next upstream device and test ACS up to the root bus.
4412 * Finding the next device may require skipping virtual buses.
4414 while (!pci_is_root_bus(dma_pdev->bus)) {
4415 struct pci_bus *bus = dma_pdev->bus;
4417 while (!bus->self) {
4418 if (!pci_is_root_bus(bus))
4424 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4427 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4431 group = iommu_group_get(&dma_pdev->dev);
4432 pci_dev_put(dma_pdev);
4434 group = iommu_group_alloc();
4436 return PTR_ERR(group);
4439 ret = iommu_group_add_device(group, dev);
4441 iommu_group_put(group);
4445 static void intel_iommu_remove_device(struct device *dev)
4447 iommu_group_remove_device(dev);
4450 static struct iommu_ops intel_iommu_ops = {
4451 .domain_init = intel_iommu_domain_init,
4452 .domain_destroy = intel_iommu_domain_destroy,
4453 .attach_dev = intel_iommu_attach_device,
4454 .detach_dev = intel_iommu_detach_device,
4455 .map = intel_iommu_map,
4456 .unmap = intel_iommu_unmap,
4457 .iova_to_phys = intel_iommu_iova_to_phys,
4458 .domain_has_cap = intel_iommu_domain_has_cap,
4459 .add_device = intel_iommu_add_device,
4460 .remove_device = intel_iommu_remove_device,
4461 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4464 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4466 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4467 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4471 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4472 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4473 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4474 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4475 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4476 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4477 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4479 static void quirk_iommu_rwbf(struct pci_dev *dev)
4482 * Mobile 4 Series Chipset neglects to set RWBF capability,
4483 * but needs it. Same seems to hold for the desktop versions.
4485 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4489 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4490 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4491 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4492 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4493 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4494 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4495 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4498 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4499 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4500 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4501 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4502 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4503 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4504 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4505 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4507 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4511 if (pci_read_config_word(dev, GGC, &ggc))
4514 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4515 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4517 } else if (dmar_map_gfx) {
4518 /* we have to ensure the gfx device is idle before we flush */
4519 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4520 intel_iommu_strict = 1;
4523 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4524 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4525 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4526 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4528 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4529 ISOCH DMAR unit for the Azalia sound device, but not give it any
4530 TLB entries, which causes it to deadlock. Check for that. We do
4531 this in a function called from init_dmars(), instead of in a PCI
4532 quirk, because we don't want to print the obnoxious "BIOS broken"
4533 message if VT-d is actually disabled.
4535 static void __init check_tylersburg_isoch(void)
4537 struct pci_dev *pdev;
4538 uint32_t vtisochctrl;
4540 /* If there's no Azalia in the system anyway, forget it. */
4541 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4546 /* System Management Registers. Might be hidden, in which case
4547 we can't do the sanity check. But that's OK, because the
4548 known-broken BIOSes _don't_ actually hide it, so far. */
4549 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4553 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4560 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4561 if (vtisochctrl & 1)
4564 /* Drop all bits other than the number of TLB entries */
4565 vtisochctrl &= 0x1c;
4567 /* If we have the recommended number of TLB entries (16), fine. */
4568 if (vtisochctrl == 0x10)
4571 /* Zero TLB entries? You get to ride the short bus to school. */
4573 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4574 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4575 dmi_get_system_info(DMI_BIOS_VENDOR),
4576 dmi_get_system_info(DMI_BIOS_VERSION),
4577 dmi_get_system_info(DMI_PRODUCT_VERSION));
4578 iommu_identity_mapping |= IDENTMAP_AZALIA;
4582 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",