2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/syscore_ops.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <linux/pci-ats.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
47 #define ROOT_SIZE VTD_PAGE_SIZE
48 #define CONTEXT_SIZE VTD_PAGE_SIZE
50 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
51 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
52 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
54 #define IOAPIC_RANGE_START (0xfee00000)
55 #define IOAPIC_RANGE_END (0xfeefffff)
56 #define IOVA_START_ADDR (0x1000)
58 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
60 #define MAX_AGAW_WIDTH 64
62 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
63 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
65 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
66 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
67 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
68 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
69 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
71 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
72 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
73 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
75 /* page table handling */
76 #define LEVEL_STRIDE (9)
77 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
79 static inline int agaw_to_level(int agaw)
84 static inline int agaw_to_width(int agaw)
86 return 30 + agaw * LEVEL_STRIDE;
89 static inline int width_to_agaw(int width)
91 return (width - 30) / LEVEL_STRIDE;
94 static inline unsigned int level_to_offset_bits(int level)
96 return (level - 1) * LEVEL_STRIDE;
99 static inline int pfn_level_offset(unsigned long pfn, int level)
101 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
104 static inline unsigned long level_mask(int level)
106 return -1UL << level_to_offset_bits(level);
109 static inline unsigned long level_size(int level)
111 return 1UL << level_to_offset_bits(level);
114 static inline unsigned long align_to_level(unsigned long pfn, int level)
116 return (pfn + level_size(level) - 1) & level_mask(level);
119 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
120 are never going to work. */
121 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
123 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
126 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
128 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
130 static inline unsigned long page_to_dma_pfn(struct page *pg)
132 return mm_to_dma_pfn(page_to_pfn(pg));
134 static inline unsigned long virt_to_dma_pfn(void *p)
136 return page_to_dma_pfn(virt_to_page(p));
139 /* global iommu list, set NULL for ignored DMAR units */
140 static struct intel_iommu **g_iommus;
142 static void __init check_tylersburg_isoch(void);
143 static int rwbf_quirk;
148 * 12-63: Context Ptr (12 - (haw-1))
155 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
156 static inline bool root_present(struct root_entry *root)
158 return (root->val & 1);
160 static inline void set_root_present(struct root_entry *root)
164 static inline void set_root_value(struct root_entry *root, unsigned long value)
166 root->val |= value & VTD_PAGE_MASK;
169 static inline struct context_entry *
170 get_context_addr_from_root(struct root_entry *root)
172 return (struct context_entry *)
173 (root_present(root)?phys_to_virt(
174 root->val & VTD_PAGE_MASK) :
181 * 1: fault processing disable
182 * 2-3: translation type
183 * 12-63: address space root
189 struct context_entry {
194 static inline bool context_present(struct context_entry *context)
196 return (context->lo & 1);
198 static inline void context_set_present(struct context_entry *context)
203 static inline void context_set_fault_enable(struct context_entry *context)
205 context->lo &= (((u64)-1) << 2) | 1;
208 static inline void context_set_translation_type(struct context_entry *context,
211 context->lo &= (((u64)-1) << 4) | 3;
212 context->lo |= (value & 3) << 2;
215 static inline void context_set_address_root(struct context_entry *context,
218 context->lo |= value & VTD_PAGE_MASK;
221 static inline void context_set_address_width(struct context_entry *context,
224 context->hi |= value & 7;
227 static inline void context_set_domain_id(struct context_entry *context,
230 context->hi |= (value & ((1 << 16) - 1)) << 8;
233 static inline void context_clear_entry(struct context_entry *context)
246 * 12-63: Host physcial address
252 static inline void dma_clear_pte(struct dma_pte *pte)
257 static inline void dma_set_pte_readable(struct dma_pte *pte)
259 pte->val |= DMA_PTE_READ;
262 static inline void dma_set_pte_writable(struct dma_pte *pte)
264 pte->val |= DMA_PTE_WRITE;
267 static inline void dma_set_pte_snp(struct dma_pte *pte)
269 pte->val |= DMA_PTE_SNP;
272 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
274 pte->val = (pte->val & ~3) | (prot & 3);
277 static inline u64 dma_pte_addr(struct dma_pte *pte)
280 return pte->val & VTD_PAGE_MASK;
282 /* Must have a full atomic 64-bit read */
283 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
287 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
289 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
292 static inline bool dma_pte_present(struct dma_pte *pte)
294 return (pte->val & 3) != 0;
297 static inline int first_pte_in_page(struct dma_pte *pte)
299 return !((unsigned long)pte & ~VTD_PAGE_MASK);
303 * This domain is a statically identity mapping domain.
304 * 1. This domain creats a static 1:1 mapping to all usable memory.
305 * 2. It maps to each iommu if successful.
306 * 3. Each iommu mapps to this domain if successful.
308 static struct dmar_domain *si_domain;
309 static int hw_pass_through = 1;
311 /* devices under the same p2p bridge are owned in one domain */
312 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
314 /* domain represents a virtual machine, more than one devices
315 * across iommus may be owned in one domain, e.g. kvm guest.
317 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
319 /* si_domain contains mulitple devices */
320 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
323 int id; /* domain id */
324 int nid; /* node id */
325 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
327 struct list_head devices; /* all devices' list */
328 struct iova_domain iovad; /* iova's that belong to this domain */
330 struct dma_pte *pgd; /* virtual address */
331 int gaw; /* max guest address width */
333 /* adjusted guest address width, 0 is level 2 30-bit */
336 int flags; /* flags to find out type of domain */
338 int iommu_coherency;/* indicate coherency of iommu access */
339 int iommu_snooping; /* indicate snooping control feature*/
340 int iommu_count; /* reference count of iommu */
341 spinlock_t iommu_lock; /* protect iommu set in domain */
342 u64 max_addr; /* maximum mapped address */
345 /* PCI domain-device relationship */
346 struct device_domain_info {
347 struct list_head link; /* link to domain siblings */
348 struct list_head global; /* link to global list */
349 int segment; /* PCI domain */
350 u8 bus; /* PCI bus number */
351 u8 devfn; /* PCI devfn number */
352 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
353 struct intel_iommu *iommu; /* IOMMU used by this device */
354 struct dmar_domain *domain; /* pointer to domain */
357 static void flush_unmaps_timeout(unsigned long data);
359 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
361 #define HIGH_WATER_MARK 250
362 struct deferred_flush_tables {
364 struct iova *iova[HIGH_WATER_MARK];
365 struct dmar_domain *domain[HIGH_WATER_MARK];
368 static struct deferred_flush_tables *deferred_flush;
370 /* bitmap for indexing intel_iommus */
371 static int g_num_of_iommus;
373 static DEFINE_SPINLOCK(async_umap_flush_lock);
374 static LIST_HEAD(unmaps_to_do);
377 static long list_size;
379 static void domain_remove_dev_info(struct dmar_domain *domain);
381 #ifdef CONFIG_DMAR_DEFAULT_ON
382 int dmar_disabled = 0;
384 int dmar_disabled = 1;
385 #endif /*CONFIG_DMAR_DEFAULT_ON*/
387 static int dmar_map_gfx = 1;
388 static int dmar_forcedac;
389 static int intel_iommu_strict;
391 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
392 static DEFINE_SPINLOCK(device_domain_lock);
393 static LIST_HEAD(device_domain_list);
395 static struct iommu_ops intel_iommu_ops;
397 static int __init intel_iommu_setup(char *str)
402 if (!strncmp(str, "on", 2)) {
404 printk(KERN_INFO "Intel-IOMMU: enabled\n");
405 } else if (!strncmp(str, "off", 3)) {
407 printk(KERN_INFO "Intel-IOMMU: disabled\n");
408 } else if (!strncmp(str, "igfx_off", 8)) {
411 "Intel-IOMMU: disable GFX device mapping\n");
412 } else if (!strncmp(str, "forcedac", 8)) {
414 "Intel-IOMMU: Forcing DAC for PCI devices\n");
416 } else if (!strncmp(str, "strict", 6)) {
418 "Intel-IOMMU: disable batched IOTLB flush\n");
419 intel_iommu_strict = 1;
422 str += strcspn(str, ",");
428 __setup("intel_iommu=", intel_iommu_setup);
430 static struct kmem_cache *iommu_domain_cache;
431 static struct kmem_cache *iommu_devinfo_cache;
432 static struct kmem_cache *iommu_iova_cache;
434 static inline void *alloc_pgtable_page(int node)
439 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
441 vaddr = page_address(page);
445 static inline void free_pgtable_page(void *vaddr)
447 free_page((unsigned long)vaddr);
450 static inline void *alloc_domain_mem(void)
452 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
455 static void free_domain_mem(void *vaddr)
457 kmem_cache_free(iommu_domain_cache, vaddr);
460 static inline void * alloc_devinfo_mem(void)
462 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
465 static inline void free_devinfo_mem(void *vaddr)
467 kmem_cache_free(iommu_devinfo_cache, vaddr);
470 struct iova *alloc_iova_mem(void)
472 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
475 void free_iova_mem(struct iova *iova)
477 kmem_cache_free(iommu_iova_cache, iova);
481 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
486 sagaw = cap_sagaw(iommu->cap);
487 for (agaw = width_to_agaw(max_gaw);
489 if (test_bit(agaw, &sagaw))
497 * Calculate max SAGAW for each iommu.
499 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
501 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
505 * calculate agaw for each iommu.
506 * "SAGAW" may be different across iommus, use a default agaw, and
507 * get a supported less agaw for iommus that don't support the default agaw.
509 int iommu_calculate_agaw(struct intel_iommu *iommu)
511 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
514 /* This functionin only returns single iommu in a domain */
515 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
519 /* si_domain and vm domain should not get here. */
520 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
521 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
523 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
524 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
527 return g_iommus[iommu_id];
530 static void domain_update_iommu_coherency(struct dmar_domain *domain)
534 domain->iommu_coherency = 1;
536 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
537 if (!ecap_coherent(g_iommus[i]->ecap)) {
538 domain->iommu_coherency = 0;
544 static void domain_update_iommu_snooping(struct dmar_domain *domain)
548 domain->iommu_snooping = 1;
550 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
551 if (!ecap_sc_support(g_iommus[i]->ecap)) {
552 domain->iommu_snooping = 0;
558 /* Some capabilities may be different across iommus */
559 static void domain_update_iommu_cap(struct dmar_domain *domain)
561 domain_update_iommu_coherency(domain);
562 domain_update_iommu_snooping(domain);
565 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
567 struct dmar_drhd_unit *drhd = NULL;
570 for_each_drhd_unit(drhd) {
573 if (segment != drhd->segment)
576 for (i = 0; i < drhd->devices_cnt; i++) {
577 if (drhd->devices[i] &&
578 drhd->devices[i]->bus->number == bus &&
579 drhd->devices[i]->devfn == devfn)
581 if (drhd->devices[i] &&
582 drhd->devices[i]->subordinate &&
583 drhd->devices[i]->subordinate->number <= bus &&
584 drhd->devices[i]->subordinate->subordinate >= bus)
588 if (drhd->include_all)
595 static void domain_flush_cache(struct dmar_domain *domain,
596 void *addr, int size)
598 if (!domain->iommu_coherency)
599 clflush_cache_range(addr, size);
602 /* Gets context entry for a given bus and devfn */
603 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
606 struct root_entry *root;
607 struct context_entry *context;
608 unsigned long phy_addr;
611 spin_lock_irqsave(&iommu->lock, flags);
612 root = &iommu->root_entry[bus];
613 context = get_context_addr_from_root(root);
615 context = (struct context_entry *)
616 alloc_pgtable_page(iommu->node);
618 spin_unlock_irqrestore(&iommu->lock, flags);
621 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
622 phy_addr = virt_to_phys((void *)context);
623 set_root_value(root, phy_addr);
624 set_root_present(root);
625 __iommu_flush_cache(iommu, root, sizeof(*root));
627 spin_unlock_irqrestore(&iommu->lock, flags);
628 return &context[devfn];
631 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
633 struct root_entry *root;
634 struct context_entry *context;
638 spin_lock_irqsave(&iommu->lock, flags);
639 root = &iommu->root_entry[bus];
640 context = get_context_addr_from_root(root);
645 ret = context_present(&context[devfn]);
647 spin_unlock_irqrestore(&iommu->lock, flags);
651 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
653 struct root_entry *root;
654 struct context_entry *context;
657 spin_lock_irqsave(&iommu->lock, flags);
658 root = &iommu->root_entry[bus];
659 context = get_context_addr_from_root(root);
661 context_clear_entry(&context[devfn]);
662 __iommu_flush_cache(iommu, &context[devfn], \
665 spin_unlock_irqrestore(&iommu->lock, flags);
668 static void free_context_table(struct intel_iommu *iommu)
670 struct root_entry *root;
673 struct context_entry *context;
675 spin_lock_irqsave(&iommu->lock, flags);
676 if (!iommu->root_entry) {
679 for (i = 0; i < ROOT_ENTRY_NR; i++) {
680 root = &iommu->root_entry[i];
681 context = get_context_addr_from_root(root);
683 free_pgtable_page(context);
685 free_pgtable_page(iommu->root_entry);
686 iommu->root_entry = NULL;
688 spin_unlock_irqrestore(&iommu->lock, flags);
691 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
694 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
695 struct dma_pte *parent, *pte = NULL;
696 int level = agaw_to_level(domain->agaw);
699 BUG_ON(!domain->pgd);
700 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
701 parent = domain->pgd;
706 offset = pfn_level_offset(pfn, level);
707 pte = &parent[offset];
711 if (!dma_pte_present(pte)) {
714 tmp_page = alloc_pgtable_page(domain->nid);
719 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
720 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
721 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
722 /* Someone else set it while we were thinking; use theirs. */
723 free_pgtable_page(tmp_page);
726 domain_flush_cache(domain, pte, sizeof(*pte));
729 parent = phys_to_virt(dma_pte_addr(pte));
736 /* return address's pte at specific level */
737 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
741 struct dma_pte *parent, *pte = NULL;
742 int total = agaw_to_level(domain->agaw);
745 parent = domain->pgd;
746 while (level <= total) {
747 offset = pfn_level_offset(pfn, total);
748 pte = &parent[offset];
752 if (!dma_pte_present(pte))
754 parent = phys_to_virt(dma_pte_addr(pte));
760 /* clear last level pte, a tlb flush should be followed */
761 static void dma_pte_clear_range(struct dmar_domain *domain,
762 unsigned long start_pfn,
763 unsigned long last_pfn)
765 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
766 struct dma_pte *first_pte, *pte;
768 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
769 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
770 BUG_ON(start_pfn > last_pfn);
772 /* we don't need lock here; nobody else touches the iova range */
774 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
776 start_pfn = align_to_level(start_pfn + 1, 2);
783 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
785 domain_flush_cache(domain, first_pte,
786 (void *)pte - (void *)first_pte);
788 } while (start_pfn && start_pfn <= last_pfn);
791 /* free page table pages. last level pte should already be cleared */
792 static void dma_pte_free_pagetable(struct dmar_domain *domain,
793 unsigned long start_pfn,
794 unsigned long last_pfn)
796 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
797 struct dma_pte *first_pte, *pte;
798 int total = agaw_to_level(domain->agaw);
802 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
803 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
804 BUG_ON(start_pfn > last_pfn);
806 /* We don't need lock here; nobody else touches the iova range */
808 while (level <= total) {
809 tmp = align_to_level(start_pfn, level);
811 /* If we can't even clear one PTE at this level, we're done */
812 if (tmp + level_size(level) - 1 > last_pfn)
816 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
818 tmp = align_to_level(tmp + 1, level + 1);
822 if (dma_pte_present(pte)) {
823 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
827 tmp += level_size(level);
828 } while (!first_pte_in_page(pte) &&
829 tmp + level_size(level) - 1 <= last_pfn);
831 domain_flush_cache(domain, first_pte,
832 (void *)pte - (void *)first_pte);
834 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
838 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
839 free_pgtable_page(domain->pgd);
845 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
847 struct root_entry *root;
850 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
854 __iommu_flush_cache(iommu, root, ROOT_SIZE);
856 spin_lock_irqsave(&iommu->lock, flags);
857 iommu->root_entry = root;
858 spin_unlock_irqrestore(&iommu->lock, flags);
863 static void iommu_set_root_entry(struct intel_iommu *iommu)
869 addr = iommu->root_entry;
871 spin_lock_irqsave(&iommu->register_lock, flag);
872 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
874 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
876 /* Make sure hardware complete it */
877 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
878 readl, (sts & DMA_GSTS_RTPS), sts);
880 spin_unlock_irqrestore(&iommu->register_lock, flag);
883 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
888 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
891 spin_lock_irqsave(&iommu->register_lock, flag);
892 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
894 /* Make sure hardware complete it */
895 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
896 readl, (!(val & DMA_GSTS_WBFS)), val);
898 spin_unlock_irqrestore(&iommu->register_lock, flag);
901 /* return value determine if we need a write buffer flush */
902 static void __iommu_flush_context(struct intel_iommu *iommu,
903 u16 did, u16 source_id, u8 function_mask,
910 case DMA_CCMD_GLOBAL_INVL:
911 val = DMA_CCMD_GLOBAL_INVL;
913 case DMA_CCMD_DOMAIN_INVL:
914 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
916 case DMA_CCMD_DEVICE_INVL:
917 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
918 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
925 spin_lock_irqsave(&iommu->register_lock, flag);
926 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
928 /* Make sure hardware complete it */
929 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
930 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
932 spin_unlock_irqrestore(&iommu->register_lock, flag);
935 /* return value determine if we need a write buffer flush */
936 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
937 u64 addr, unsigned int size_order, u64 type)
939 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
940 u64 val = 0, val_iva = 0;
944 case DMA_TLB_GLOBAL_FLUSH:
945 /* global flush doesn't need set IVA_REG */
946 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
948 case DMA_TLB_DSI_FLUSH:
949 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
951 case DMA_TLB_PSI_FLUSH:
952 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
953 /* Note: always flush non-leaf currently */
954 val_iva = size_order | addr;
959 /* Note: set drain read/write */
962 * This is probably to be super secure.. Looks like we can
963 * ignore it without any impact.
965 if (cap_read_drain(iommu->cap))
966 val |= DMA_TLB_READ_DRAIN;
968 if (cap_write_drain(iommu->cap))
969 val |= DMA_TLB_WRITE_DRAIN;
971 spin_lock_irqsave(&iommu->register_lock, flag);
972 /* Note: Only uses first TLB reg currently */
974 dmar_writeq(iommu->reg + tlb_offset, val_iva);
975 dmar_writeq(iommu->reg + tlb_offset + 8, val);
977 /* Make sure hardware complete it */
978 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
979 dmar_readq, (!(val & DMA_TLB_IVT)), val);
981 spin_unlock_irqrestore(&iommu->register_lock, flag);
983 /* check IOTLB invalidation granularity */
984 if (DMA_TLB_IAIG(val) == 0)
985 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
986 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
987 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
988 (unsigned long long)DMA_TLB_IIRG(type),
989 (unsigned long long)DMA_TLB_IAIG(val));
992 static struct device_domain_info *iommu_support_dev_iotlb(
993 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
997 struct device_domain_info *info;
998 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1000 if (!ecap_dev_iotlb_support(iommu->ecap))
1006 spin_lock_irqsave(&device_domain_lock, flags);
1007 list_for_each_entry(info, &domain->devices, link)
1008 if (info->bus == bus && info->devfn == devfn) {
1012 spin_unlock_irqrestore(&device_domain_lock, flags);
1014 if (!found || !info->dev)
1017 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1020 if (!dmar_find_matched_atsr_unit(info->dev))
1023 info->iommu = iommu;
1028 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1033 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1036 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1038 if (!info->dev || !pci_ats_enabled(info->dev))
1041 pci_disable_ats(info->dev);
1044 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1045 u64 addr, unsigned mask)
1048 unsigned long flags;
1049 struct device_domain_info *info;
1051 spin_lock_irqsave(&device_domain_lock, flags);
1052 list_for_each_entry(info, &domain->devices, link) {
1053 if (!info->dev || !pci_ats_enabled(info->dev))
1056 sid = info->bus << 8 | info->devfn;
1057 qdep = pci_ats_queue_depth(info->dev);
1058 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1060 spin_unlock_irqrestore(&device_domain_lock, flags);
1063 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1064 unsigned long pfn, unsigned int pages, int map)
1066 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1067 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1072 * Fallback to domain selective flush if no PSI support or the size is
1074 * PSI requires page size to be 2 ^ x, and the base address is naturally
1075 * aligned to the size
1077 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1078 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1081 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1085 * In caching mode, changes of pages from non-present to present require
1086 * flush. However, device IOTLB doesn't need to be flushed in this case.
1088 if (!cap_caching_mode(iommu->cap) || !map)
1089 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1092 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1095 unsigned long flags;
1097 spin_lock_irqsave(&iommu->register_lock, flags);
1098 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1099 pmen &= ~DMA_PMEN_EPM;
1100 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1102 /* wait for the protected region status bit to clear */
1103 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1104 readl, !(pmen & DMA_PMEN_PRS), pmen);
1106 spin_unlock_irqrestore(&iommu->register_lock, flags);
1109 static int iommu_enable_translation(struct intel_iommu *iommu)
1112 unsigned long flags;
1114 spin_lock_irqsave(&iommu->register_lock, flags);
1115 iommu->gcmd |= DMA_GCMD_TE;
1116 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1118 /* Make sure hardware complete it */
1119 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1120 readl, (sts & DMA_GSTS_TES), sts);
1122 spin_unlock_irqrestore(&iommu->register_lock, flags);
1126 static int iommu_disable_translation(struct intel_iommu *iommu)
1131 spin_lock_irqsave(&iommu->register_lock, flag);
1132 iommu->gcmd &= ~DMA_GCMD_TE;
1133 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1135 /* Make sure hardware complete it */
1136 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1137 readl, (!(sts & DMA_GSTS_TES)), sts);
1139 spin_unlock_irqrestore(&iommu->register_lock, flag);
1144 static int iommu_init_domains(struct intel_iommu *iommu)
1146 unsigned long ndomains;
1147 unsigned long nlongs;
1149 ndomains = cap_ndoms(iommu->cap);
1150 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1152 nlongs = BITS_TO_LONGS(ndomains);
1154 spin_lock_init(&iommu->lock);
1156 /* TBD: there might be 64K domains,
1157 * consider other allocation for future chip
1159 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1160 if (!iommu->domain_ids) {
1161 printk(KERN_ERR "Allocating domain id array failed\n");
1164 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1166 if (!iommu->domains) {
1167 printk(KERN_ERR "Allocating domain array failed\n");
1172 * if Caching mode is set, then invalid translations are tagged
1173 * with domainid 0. Hence we need to pre-allocate it.
1175 if (cap_caching_mode(iommu->cap))
1176 set_bit(0, iommu->domain_ids);
1181 static void domain_exit(struct dmar_domain *domain);
1182 static void vm_domain_exit(struct dmar_domain *domain);
1184 void free_dmar_iommu(struct intel_iommu *iommu)
1186 struct dmar_domain *domain;
1188 unsigned long flags;
1190 if ((iommu->domains) && (iommu->domain_ids)) {
1191 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1192 domain = iommu->domains[i];
1193 clear_bit(i, iommu->domain_ids);
1195 spin_lock_irqsave(&domain->iommu_lock, flags);
1196 if (--domain->iommu_count == 0) {
1197 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1198 vm_domain_exit(domain);
1200 domain_exit(domain);
1202 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1206 if (iommu->gcmd & DMA_GCMD_TE)
1207 iommu_disable_translation(iommu);
1210 irq_set_handler_data(iommu->irq, NULL);
1211 /* This will mask the irq */
1212 free_irq(iommu->irq, iommu);
1213 destroy_irq(iommu->irq);
1216 kfree(iommu->domains);
1217 kfree(iommu->domain_ids);
1219 g_iommus[iommu->seq_id] = NULL;
1221 /* if all iommus are freed, free g_iommus */
1222 for (i = 0; i < g_num_of_iommus; i++) {
1227 if (i == g_num_of_iommus)
1230 /* free context mapping */
1231 free_context_table(iommu);
1234 static struct dmar_domain *alloc_domain(void)
1236 struct dmar_domain *domain;
1238 domain = alloc_domain_mem();
1243 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1249 static int iommu_attach_domain(struct dmar_domain *domain,
1250 struct intel_iommu *iommu)
1253 unsigned long ndomains;
1254 unsigned long flags;
1256 ndomains = cap_ndoms(iommu->cap);
1258 spin_lock_irqsave(&iommu->lock, flags);
1260 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1261 if (num >= ndomains) {
1262 spin_unlock_irqrestore(&iommu->lock, flags);
1263 printk(KERN_ERR "IOMMU: no free domain ids\n");
1268 set_bit(num, iommu->domain_ids);
1269 set_bit(iommu->seq_id, &domain->iommu_bmp);
1270 iommu->domains[num] = domain;
1271 spin_unlock_irqrestore(&iommu->lock, flags);
1276 static void iommu_detach_domain(struct dmar_domain *domain,
1277 struct intel_iommu *iommu)
1279 unsigned long flags;
1283 spin_lock_irqsave(&iommu->lock, flags);
1284 ndomains = cap_ndoms(iommu->cap);
1285 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1286 if (iommu->domains[num] == domain) {
1293 clear_bit(num, iommu->domain_ids);
1294 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1295 iommu->domains[num] = NULL;
1297 spin_unlock_irqrestore(&iommu->lock, flags);
1300 static struct iova_domain reserved_iova_list;
1301 static struct lock_class_key reserved_rbtree_key;
1303 static int dmar_init_reserved_ranges(void)
1305 struct pci_dev *pdev = NULL;
1309 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1311 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1312 &reserved_rbtree_key);
1314 /* IOAPIC ranges shouldn't be accessed by DMA */
1315 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1316 IOVA_PFN(IOAPIC_RANGE_END));
1318 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1322 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1323 for_each_pci_dev(pdev) {
1326 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1327 r = &pdev->resource[i];
1328 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1330 iova = reserve_iova(&reserved_iova_list,
1334 printk(KERN_ERR "Reserve iova failed\n");
1342 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1344 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1347 static inline int guestwidth_to_adjustwidth(int gaw)
1350 int r = (gaw - 12) % 9;
1361 static int domain_init(struct dmar_domain *domain, int guest_width)
1363 struct intel_iommu *iommu;
1364 int adjust_width, agaw;
1365 unsigned long sagaw;
1367 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1368 spin_lock_init(&domain->iommu_lock);
1370 domain_reserve_special_ranges(domain);
1372 /* calculate AGAW */
1373 iommu = domain_get_iommu(domain);
1374 if (guest_width > cap_mgaw(iommu->cap))
1375 guest_width = cap_mgaw(iommu->cap);
1376 domain->gaw = guest_width;
1377 adjust_width = guestwidth_to_adjustwidth(guest_width);
1378 agaw = width_to_agaw(adjust_width);
1379 sagaw = cap_sagaw(iommu->cap);
1380 if (!test_bit(agaw, &sagaw)) {
1381 /* hardware doesn't support it, choose a bigger one */
1382 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1383 agaw = find_next_bit(&sagaw, 5, agaw);
1387 domain->agaw = agaw;
1388 INIT_LIST_HEAD(&domain->devices);
1390 if (ecap_coherent(iommu->ecap))
1391 domain->iommu_coherency = 1;
1393 domain->iommu_coherency = 0;
1395 if (ecap_sc_support(iommu->ecap))
1396 domain->iommu_snooping = 1;
1398 domain->iommu_snooping = 0;
1400 domain->iommu_count = 1;
1401 domain->nid = iommu->node;
1403 /* always allocate the top pgd */
1404 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1407 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1411 static void domain_exit(struct dmar_domain *domain)
1413 struct dmar_drhd_unit *drhd;
1414 struct intel_iommu *iommu;
1416 /* Domain 0 is reserved, so dont process it */
1420 domain_remove_dev_info(domain);
1422 put_iova_domain(&domain->iovad);
1425 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1427 /* free page tables */
1428 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1430 for_each_active_iommu(iommu, drhd)
1431 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1432 iommu_detach_domain(domain, iommu);
1434 free_domain_mem(domain);
1437 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1438 u8 bus, u8 devfn, int translation)
1440 struct context_entry *context;
1441 unsigned long flags;
1442 struct intel_iommu *iommu;
1443 struct dma_pte *pgd;
1445 unsigned long ndomains;
1448 struct device_domain_info *info = NULL;
1450 pr_debug("Set context mapping for %02x:%02x.%d\n",
1451 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1453 BUG_ON(!domain->pgd);
1454 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1455 translation != CONTEXT_TT_MULTI_LEVEL);
1457 iommu = device_to_iommu(segment, bus, devfn);
1461 context = device_to_context_entry(iommu, bus, devfn);
1464 spin_lock_irqsave(&iommu->lock, flags);
1465 if (context_present(context)) {
1466 spin_unlock_irqrestore(&iommu->lock, flags);
1473 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1474 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1477 /* find an available domain id for this device in iommu */
1478 ndomains = cap_ndoms(iommu->cap);
1479 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1480 if (iommu->domains[num] == domain) {
1488 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1489 if (num >= ndomains) {
1490 spin_unlock_irqrestore(&iommu->lock, flags);
1491 printk(KERN_ERR "IOMMU: no free domain ids\n");
1495 set_bit(num, iommu->domain_ids);
1496 iommu->domains[num] = domain;
1500 /* Skip top levels of page tables for
1501 * iommu which has less agaw than default.
1502 * Unnecessary for PT mode.
1504 if (translation != CONTEXT_TT_PASS_THROUGH) {
1505 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1506 pgd = phys_to_virt(dma_pte_addr(pgd));
1507 if (!dma_pte_present(pgd)) {
1508 spin_unlock_irqrestore(&iommu->lock, flags);
1515 context_set_domain_id(context, id);
1517 if (translation != CONTEXT_TT_PASS_THROUGH) {
1518 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1519 translation = info ? CONTEXT_TT_DEV_IOTLB :
1520 CONTEXT_TT_MULTI_LEVEL;
1523 * In pass through mode, AW must be programmed to indicate the largest
1524 * AGAW value supported by hardware. And ASR is ignored by hardware.
1526 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1527 context_set_address_width(context, iommu->msagaw);
1529 context_set_address_root(context, virt_to_phys(pgd));
1530 context_set_address_width(context, iommu->agaw);
1533 context_set_translation_type(context, translation);
1534 context_set_fault_enable(context);
1535 context_set_present(context);
1536 domain_flush_cache(domain, context, sizeof(*context));
1539 * It's a non-present to present mapping. If hardware doesn't cache
1540 * non-present entry we only need to flush the write-buffer. If the
1541 * _does_ cache non-present entries, then it does so in the special
1542 * domain #0, which we have to flush:
1544 if (cap_caching_mode(iommu->cap)) {
1545 iommu->flush.flush_context(iommu, 0,
1546 (((u16)bus) << 8) | devfn,
1547 DMA_CCMD_MASK_NOBIT,
1548 DMA_CCMD_DEVICE_INVL);
1549 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1551 iommu_flush_write_buffer(iommu);
1553 iommu_enable_dev_iotlb(info);
1554 spin_unlock_irqrestore(&iommu->lock, flags);
1556 spin_lock_irqsave(&domain->iommu_lock, flags);
1557 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1558 domain->iommu_count++;
1559 if (domain->iommu_count == 1)
1560 domain->nid = iommu->node;
1561 domain_update_iommu_cap(domain);
1563 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1568 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1572 struct pci_dev *tmp, *parent;
1574 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1575 pdev->bus->number, pdev->devfn,
1580 /* dependent device mapping */
1581 tmp = pci_find_upstream_pcie_bridge(pdev);
1584 /* Secondary interface's bus number and devfn 0 */
1585 parent = pdev->bus->self;
1586 while (parent != tmp) {
1587 ret = domain_context_mapping_one(domain,
1588 pci_domain_nr(parent->bus),
1589 parent->bus->number,
1590 parent->devfn, translation);
1593 parent = parent->bus->self;
1595 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1596 return domain_context_mapping_one(domain,
1597 pci_domain_nr(tmp->subordinate),
1598 tmp->subordinate->number, 0,
1600 else /* this is a legacy PCI bridge */
1601 return domain_context_mapping_one(domain,
1602 pci_domain_nr(tmp->bus),
1608 static int domain_context_mapped(struct pci_dev *pdev)
1611 struct pci_dev *tmp, *parent;
1612 struct intel_iommu *iommu;
1614 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1619 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1622 /* dependent device mapping */
1623 tmp = pci_find_upstream_pcie_bridge(pdev);
1626 /* Secondary interface's bus number and devfn 0 */
1627 parent = pdev->bus->self;
1628 while (parent != tmp) {
1629 ret = device_context_mapped(iommu, parent->bus->number,
1633 parent = parent->bus->self;
1635 if (pci_is_pcie(tmp))
1636 return device_context_mapped(iommu, tmp->subordinate->number,
1639 return device_context_mapped(iommu, tmp->bus->number,
1643 /* Returns a number of VTD pages, but aligned to MM page size */
1644 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1647 host_addr &= ~PAGE_MASK;
1648 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1651 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1652 struct scatterlist *sg, unsigned long phys_pfn,
1653 unsigned long nr_pages, int prot)
1655 struct dma_pte *first_pte = NULL, *pte = NULL;
1656 phys_addr_t uninitialized_var(pteval);
1657 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1658 unsigned long sg_res;
1660 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1662 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1665 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1670 sg_res = nr_pages + 1;
1671 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1674 while (nr_pages--) {
1678 sg_res = aligned_nrpages(sg->offset, sg->length);
1679 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1680 sg->dma_length = sg->length;
1681 pteval = page_to_phys(sg_page(sg)) | prot;
1684 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1688 /* We don't need lock here, nobody else
1689 * touches the iova range
1691 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1693 static int dumps = 5;
1694 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1695 iov_pfn, tmp, (unsigned long long)pteval);
1698 debug_dma_dump_mappings(NULL);
1703 if (!nr_pages || first_pte_in_page(pte)) {
1704 domain_flush_cache(domain, first_pte,
1705 (void *)pte - (void *)first_pte);
1709 pteval += VTD_PAGE_SIZE;
1717 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1718 struct scatterlist *sg, unsigned long nr_pages,
1721 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1724 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1725 unsigned long phys_pfn, unsigned long nr_pages,
1728 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1731 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1736 clear_context_table(iommu, bus, devfn);
1737 iommu->flush.flush_context(iommu, 0, 0, 0,
1738 DMA_CCMD_GLOBAL_INVL);
1739 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1742 static void domain_remove_dev_info(struct dmar_domain *domain)
1744 struct device_domain_info *info;
1745 unsigned long flags;
1746 struct intel_iommu *iommu;
1748 spin_lock_irqsave(&device_domain_lock, flags);
1749 while (!list_empty(&domain->devices)) {
1750 info = list_entry(domain->devices.next,
1751 struct device_domain_info, link);
1752 list_del(&info->link);
1753 list_del(&info->global);
1755 info->dev->dev.archdata.iommu = NULL;
1756 spin_unlock_irqrestore(&device_domain_lock, flags);
1758 iommu_disable_dev_iotlb(info);
1759 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1760 iommu_detach_dev(iommu, info->bus, info->devfn);
1761 free_devinfo_mem(info);
1763 spin_lock_irqsave(&device_domain_lock, flags);
1765 spin_unlock_irqrestore(&device_domain_lock, flags);
1770 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1772 static struct dmar_domain *
1773 find_domain(struct pci_dev *pdev)
1775 struct device_domain_info *info;
1777 /* No lock here, assumes no domain exit in normal case */
1778 info = pdev->dev.archdata.iommu;
1780 return info->domain;
1784 /* domain is initialized */
1785 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1787 struct dmar_domain *domain, *found = NULL;
1788 struct intel_iommu *iommu;
1789 struct dmar_drhd_unit *drhd;
1790 struct device_domain_info *info, *tmp;
1791 struct pci_dev *dev_tmp;
1792 unsigned long flags;
1793 int bus = 0, devfn = 0;
1797 domain = find_domain(pdev);
1801 segment = pci_domain_nr(pdev->bus);
1803 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1805 if (pci_is_pcie(dev_tmp)) {
1806 bus = dev_tmp->subordinate->number;
1809 bus = dev_tmp->bus->number;
1810 devfn = dev_tmp->devfn;
1812 spin_lock_irqsave(&device_domain_lock, flags);
1813 list_for_each_entry(info, &device_domain_list, global) {
1814 if (info->segment == segment &&
1815 info->bus == bus && info->devfn == devfn) {
1816 found = info->domain;
1820 spin_unlock_irqrestore(&device_domain_lock, flags);
1821 /* pcie-pci bridge already has a domain, uses it */
1828 domain = alloc_domain();
1832 /* Allocate new domain for the device */
1833 drhd = dmar_find_matched_drhd_unit(pdev);
1835 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1839 iommu = drhd->iommu;
1841 ret = iommu_attach_domain(domain, iommu);
1843 free_domain_mem(domain);
1847 if (domain_init(domain, gaw)) {
1848 domain_exit(domain);
1852 /* register pcie-to-pci device */
1854 info = alloc_devinfo_mem();
1856 domain_exit(domain);
1859 info->segment = segment;
1861 info->devfn = devfn;
1863 info->domain = domain;
1864 /* This domain is shared by devices under p2p bridge */
1865 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1867 /* pcie-to-pci bridge already has a domain, uses it */
1869 spin_lock_irqsave(&device_domain_lock, flags);
1870 list_for_each_entry(tmp, &device_domain_list, global) {
1871 if (tmp->segment == segment &&
1872 tmp->bus == bus && tmp->devfn == devfn) {
1873 found = tmp->domain;
1878 spin_unlock_irqrestore(&device_domain_lock, flags);
1879 free_devinfo_mem(info);
1880 domain_exit(domain);
1883 list_add(&info->link, &domain->devices);
1884 list_add(&info->global, &device_domain_list);
1885 spin_unlock_irqrestore(&device_domain_lock, flags);
1890 info = alloc_devinfo_mem();
1893 info->segment = segment;
1894 info->bus = pdev->bus->number;
1895 info->devfn = pdev->devfn;
1897 info->domain = domain;
1898 spin_lock_irqsave(&device_domain_lock, flags);
1899 /* somebody is fast */
1900 found = find_domain(pdev);
1901 if (found != NULL) {
1902 spin_unlock_irqrestore(&device_domain_lock, flags);
1903 if (found != domain) {
1904 domain_exit(domain);
1907 free_devinfo_mem(info);
1910 list_add(&info->link, &domain->devices);
1911 list_add(&info->global, &device_domain_list);
1912 pdev->dev.archdata.iommu = info;
1913 spin_unlock_irqrestore(&device_domain_lock, flags);
1916 /* recheck it here, maybe others set it */
1917 return find_domain(pdev);
1920 static int iommu_identity_mapping;
1921 #define IDENTMAP_ALL 1
1922 #define IDENTMAP_GFX 2
1923 #define IDENTMAP_AZALIA 4
1925 static int iommu_domain_identity_map(struct dmar_domain *domain,
1926 unsigned long long start,
1927 unsigned long long end)
1929 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1930 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1932 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1933 dma_to_mm_pfn(last_vpfn))) {
1934 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1938 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1939 start, end, domain->id);
1941 * RMRR range might have overlap with physical memory range,
1944 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1946 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1947 last_vpfn - first_vpfn + 1,
1948 DMA_PTE_READ|DMA_PTE_WRITE);
1951 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1952 unsigned long long start,
1953 unsigned long long end)
1955 struct dmar_domain *domain;
1958 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1962 /* For _hardware_ passthrough, don't bother. But for software
1963 passthrough, we do it anyway -- it may indicate a memory
1964 range which is reserved in E820, so which didn't get set
1965 up to start with in si_domain */
1966 if (domain == si_domain && hw_pass_through) {
1967 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1968 pci_name(pdev), start, end);
1973 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1974 pci_name(pdev), start, end);
1977 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1978 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1979 dmi_get_system_info(DMI_BIOS_VENDOR),
1980 dmi_get_system_info(DMI_BIOS_VERSION),
1981 dmi_get_system_info(DMI_PRODUCT_VERSION));
1986 if (end >> agaw_to_width(domain->agaw)) {
1987 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1988 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1989 agaw_to_width(domain->agaw),
1990 dmi_get_system_info(DMI_BIOS_VENDOR),
1991 dmi_get_system_info(DMI_BIOS_VERSION),
1992 dmi_get_system_info(DMI_PRODUCT_VERSION));
1997 ret = iommu_domain_identity_map(domain, start, end);
2001 /* context entry init */
2002 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2009 domain_exit(domain);
2013 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2014 struct pci_dev *pdev)
2016 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2018 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2019 rmrr->end_address + 1);
2022 #ifdef CONFIG_DMAR_FLOPPY_WA
2023 static inline void iommu_prepare_isa(void)
2025 struct pci_dev *pdev;
2028 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2032 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2033 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2036 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2037 "floppy might not work\n");
2041 static inline void iommu_prepare_isa(void)
2045 #endif /* !CONFIG_DMAR_FLPY_WA */
2047 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2049 static int __init si_domain_work_fn(unsigned long start_pfn,
2050 unsigned long end_pfn, void *datax)
2054 *ret = iommu_domain_identity_map(si_domain,
2055 (uint64_t)start_pfn << PAGE_SHIFT,
2056 (uint64_t)end_pfn << PAGE_SHIFT);
2061 static int __init si_domain_init(int hw)
2063 struct dmar_drhd_unit *drhd;
2064 struct intel_iommu *iommu;
2067 si_domain = alloc_domain();
2071 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2073 for_each_active_iommu(iommu, drhd) {
2074 ret = iommu_attach_domain(si_domain, iommu);
2076 domain_exit(si_domain);
2081 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2082 domain_exit(si_domain);
2086 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2091 for_each_online_node(nid) {
2092 work_with_active_regions(nid, si_domain_work_fn, &ret);
2100 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2101 struct pci_dev *pdev);
2102 static int identity_mapping(struct pci_dev *pdev)
2104 struct device_domain_info *info;
2106 if (likely(!iommu_identity_mapping))
2110 list_for_each_entry(info, &si_domain->devices, link)
2111 if (info->dev == pdev)
2116 static int domain_add_dev_info(struct dmar_domain *domain,
2117 struct pci_dev *pdev,
2120 struct device_domain_info *info;
2121 unsigned long flags;
2124 info = alloc_devinfo_mem();
2128 ret = domain_context_mapping(domain, pdev, translation);
2130 free_devinfo_mem(info);
2134 info->segment = pci_domain_nr(pdev->bus);
2135 info->bus = pdev->bus->number;
2136 info->devfn = pdev->devfn;
2138 info->domain = domain;
2140 spin_lock_irqsave(&device_domain_lock, flags);
2141 list_add(&info->link, &domain->devices);
2142 list_add(&info->global, &device_domain_list);
2143 pdev->dev.archdata.iommu = info;
2144 spin_unlock_irqrestore(&device_domain_lock, flags);
2149 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2151 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2154 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2157 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2161 * We want to start off with all devices in the 1:1 domain, and
2162 * take them out later if we find they can't access all of memory.
2164 * However, we can't do this for PCI devices behind bridges,
2165 * because all PCI devices behind the same bridge will end up
2166 * with the same source-id on their transactions.
2168 * Practically speaking, we can't change things around for these
2169 * devices at run-time, because we can't be sure there'll be no
2170 * DMA transactions in flight for any of their siblings.
2172 * So PCI devices (unless they're on the root bus) as well as
2173 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2174 * the 1:1 domain, just in _case_ one of their siblings turns out
2175 * not to be able to map all of memory.
2177 if (!pci_is_pcie(pdev)) {
2178 if (!pci_is_root_bus(pdev->bus))
2180 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2182 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2186 * At boot time, we don't yet know if devices will be 64-bit capable.
2187 * Assume that they will -- if they turn out not to be, then we can
2188 * take them out of the 1:1 domain later.
2191 return pdev->dma_mask > DMA_BIT_MASK(32);
2196 static int __init iommu_prepare_static_identity_mapping(int hw)
2198 struct pci_dev *pdev = NULL;
2201 ret = si_domain_init(hw);
2205 for_each_pci_dev(pdev) {
2206 if (iommu_should_identity_map(pdev, 1)) {
2207 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2208 hw ? "hardware" : "software", pci_name(pdev));
2210 ret = domain_add_dev_info(si_domain, pdev,
2211 hw ? CONTEXT_TT_PASS_THROUGH :
2212 CONTEXT_TT_MULTI_LEVEL);
2221 static int __init init_dmars(int force_on)
2223 struct dmar_drhd_unit *drhd;
2224 struct dmar_rmrr_unit *rmrr;
2225 struct pci_dev *pdev;
2226 struct intel_iommu *iommu;
2232 * initialize and program root entry to not present
2235 for_each_drhd_unit(drhd) {
2238 * lock not needed as this is only incremented in the single
2239 * threaded kernel __init code path all other access are read
2244 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2247 printk(KERN_ERR "Allocating global iommu array failed\n");
2252 deferred_flush = kzalloc(g_num_of_iommus *
2253 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2254 if (!deferred_flush) {
2259 for_each_drhd_unit(drhd) {
2263 iommu = drhd->iommu;
2264 g_iommus[iommu->seq_id] = iommu;
2266 ret = iommu_init_domains(iommu);
2272 * we could share the same root & context tables
2273 * among all IOMMU's. Need to Split it later.
2275 ret = iommu_alloc_root_entry(iommu);
2277 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2280 if (!ecap_pass_through(iommu->ecap))
2281 hw_pass_through = 0;
2285 * Start from the sane iommu hardware state.
2287 for_each_drhd_unit(drhd) {
2291 iommu = drhd->iommu;
2294 * If the queued invalidation is already initialized by us
2295 * (for example, while enabling interrupt-remapping) then
2296 * we got the things already rolling from a sane state.
2302 * Clear any previous faults.
2304 dmar_fault(-1, iommu);
2306 * Disable queued invalidation if supported and already enabled
2307 * before OS handover.
2309 dmar_disable_qi(iommu);
2312 for_each_drhd_unit(drhd) {
2316 iommu = drhd->iommu;
2318 if (dmar_enable_qi(iommu)) {
2320 * Queued Invalidate not enabled, use Register Based
2323 iommu->flush.flush_context = __iommu_flush_context;
2324 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2325 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2328 (unsigned long long)drhd->reg_base_addr);
2330 iommu->flush.flush_context = qi_flush_context;
2331 iommu->flush.flush_iotlb = qi_flush_iotlb;
2332 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2335 (unsigned long long)drhd->reg_base_addr);
2339 if (iommu_pass_through)
2340 iommu_identity_mapping |= IDENTMAP_ALL;
2342 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2343 iommu_identity_mapping |= IDENTMAP_GFX;
2346 check_tylersburg_isoch();
2349 * If pass through is not set or not enabled, setup context entries for
2350 * identity mappings for rmrr, gfx, and isa and may fall back to static
2351 * identity mapping if iommu_identity_mapping is set.
2353 if (iommu_identity_mapping) {
2354 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2356 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2362 * for each dev attached to rmrr
2364 * locate drhd for dev, alloc domain for dev
2365 * allocate free domain
2366 * allocate page table entries for rmrr
2367 * if context not allocated for bus
2368 * allocate and init context
2369 * set present in root table for this bus
2370 * init context with domain, translation etc
2374 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2375 for_each_rmrr_units(rmrr) {
2376 for (i = 0; i < rmrr->devices_cnt; i++) {
2377 pdev = rmrr->devices[i];
2379 * some BIOS lists non-exist devices in DMAR
2384 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2387 "IOMMU: mapping reserved region failed\n");
2391 iommu_prepare_isa();
2396 * global invalidate context cache
2397 * global invalidate iotlb
2398 * enable translation
2400 for_each_drhd_unit(drhd) {
2401 if (drhd->ignored) {
2403 * we always have to disable PMRs or DMA may fail on
2407 iommu_disable_protect_mem_regions(drhd->iommu);
2410 iommu = drhd->iommu;
2412 iommu_flush_write_buffer(iommu);
2414 ret = dmar_set_interrupt(iommu);
2418 iommu_set_root_entry(iommu);
2420 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2421 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2423 ret = iommu_enable_translation(iommu);
2427 iommu_disable_protect_mem_regions(iommu);
2432 for_each_drhd_unit(drhd) {
2435 iommu = drhd->iommu;
2442 /* This takes a number of _MM_ pages, not VTD pages */
2443 static struct iova *intel_alloc_iova(struct device *dev,
2444 struct dmar_domain *domain,
2445 unsigned long nrpages, uint64_t dma_mask)
2447 struct pci_dev *pdev = to_pci_dev(dev);
2448 struct iova *iova = NULL;
2450 /* Restrict dma_mask to the width that the iommu can handle */
2451 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2453 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2455 * First try to allocate an io virtual address in
2456 * DMA_BIT_MASK(32) and if that fails then try allocating
2459 iova = alloc_iova(&domain->iovad, nrpages,
2460 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2464 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2465 if (unlikely(!iova)) {
2466 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2467 nrpages, pci_name(pdev));
2474 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2476 struct dmar_domain *domain;
2479 domain = get_domain_for_dev(pdev,
2480 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2483 "Allocating domain for %s failed", pci_name(pdev));
2487 /* make sure context mapping is ok */
2488 if (unlikely(!domain_context_mapped(pdev))) {
2489 ret = domain_context_mapping(domain, pdev,
2490 CONTEXT_TT_MULTI_LEVEL);
2493 "Domain context map for %s failed",
2502 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2504 struct device_domain_info *info;
2506 /* No lock here, assumes no domain exit in normal case */
2507 info = dev->dev.archdata.iommu;
2509 return info->domain;
2511 return __get_valid_domain_for_dev(dev);
2514 static int iommu_dummy(struct pci_dev *pdev)
2516 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2519 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2520 static int iommu_no_mapping(struct device *dev)
2522 struct pci_dev *pdev;
2525 if (unlikely(dev->bus != &pci_bus_type))
2528 pdev = to_pci_dev(dev);
2529 if (iommu_dummy(pdev))
2532 if (!iommu_identity_mapping)
2535 found = identity_mapping(pdev);
2537 if (iommu_should_identity_map(pdev, 0))
2541 * 32 bit DMA is removed from si_domain and fall back
2542 * to non-identity mapping.
2544 domain_remove_one_dev_info(si_domain, pdev);
2545 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2551 * In case of a detached 64 bit DMA device from vm, the device
2552 * is put into si_domain for identity mapping.
2554 if (iommu_should_identity_map(pdev, 0)) {
2556 ret = domain_add_dev_info(si_domain, pdev,
2558 CONTEXT_TT_PASS_THROUGH :
2559 CONTEXT_TT_MULTI_LEVEL);
2561 printk(KERN_INFO "64bit %s uses identity mapping\n",
2571 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2572 size_t size, int dir, u64 dma_mask)
2574 struct pci_dev *pdev = to_pci_dev(hwdev);
2575 struct dmar_domain *domain;
2576 phys_addr_t start_paddr;
2580 struct intel_iommu *iommu;
2581 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2583 BUG_ON(dir == DMA_NONE);
2585 if (iommu_no_mapping(hwdev))
2588 domain = get_valid_domain_for_dev(pdev);
2592 iommu = domain_get_iommu(domain);
2593 size = aligned_nrpages(paddr, size);
2595 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2601 * Check if DMAR supports zero-length reads on write only
2604 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2605 !cap_zlr(iommu->cap))
2606 prot |= DMA_PTE_READ;
2607 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2608 prot |= DMA_PTE_WRITE;
2610 * paddr - (paddr + size) might be partial page, we should map the whole
2611 * page. Note: if two part of one page are separately mapped, we
2612 * might have two guest_addr mapping to the same host paddr, but this
2613 * is not a big problem
2615 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2616 mm_to_dma_pfn(paddr_pfn), size, prot);
2620 /* it's a non-present to present mapping. Only flush if caching mode */
2621 if (cap_caching_mode(iommu->cap))
2622 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2624 iommu_flush_write_buffer(iommu);
2626 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2627 start_paddr += paddr & ~PAGE_MASK;
2632 __free_iova(&domain->iovad, iova);
2633 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2634 pci_name(pdev), size, (unsigned long long)paddr, dir);
2638 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2639 unsigned long offset, size_t size,
2640 enum dma_data_direction dir,
2641 struct dma_attrs *attrs)
2643 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2644 dir, to_pci_dev(dev)->dma_mask);
2647 static void flush_unmaps(void)
2653 /* just flush them all */
2654 for (i = 0; i < g_num_of_iommus; i++) {
2655 struct intel_iommu *iommu = g_iommus[i];
2659 if (!deferred_flush[i].next)
2662 /* In caching mode, global flushes turn emulation expensive */
2663 if (!cap_caching_mode(iommu->cap))
2664 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2665 DMA_TLB_GLOBAL_FLUSH);
2666 for (j = 0; j < deferred_flush[i].next; j++) {
2668 struct iova *iova = deferred_flush[i].iova[j];
2669 struct dmar_domain *domain = deferred_flush[i].domain[j];
2671 /* On real hardware multiple invalidations are expensive */
2672 if (cap_caching_mode(iommu->cap))
2673 iommu_flush_iotlb_psi(iommu, domain->id,
2674 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2676 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2677 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2678 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2680 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2682 deferred_flush[i].next = 0;
2688 static void flush_unmaps_timeout(unsigned long data)
2690 unsigned long flags;
2692 spin_lock_irqsave(&async_umap_flush_lock, flags);
2694 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2697 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2699 unsigned long flags;
2701 struct intel_iommu *iommu;
2703 spin_lock_irqsave(&async_umap_flush_lock, flags);
2704 if (list_size == HIGH_WATER_MARK)
2707 iommu = domain_get_iommu(dom);
2708 iommu_id = iommu->seq_id;
2710 next = deferred_flush[iommu_id].next;
2711 deferred_flush[iommu_id].domain[next] = dom;
2712 deferred_flush[iommu_id].iova[next] = iova;
2713 deferred_flush[iommu_id].next++;
2716 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2720 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2723 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2724 size_t size, enum dma_data_direction dir,
2725 struct dma_attrs *attrs)
2727 struct pci_dev *pdev = to_pci_dev(dev);
2728 struct dmar_domain *domain;
2729 unsigned long start_pfn, last_pfn;
2731 struct intel_iommu *iommu;
2733 if (iommu_no_mapping(dev))
2736 domain = find_domain(pdev);
2739 iommu = domain_get_iommu(domain);
2741 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2742 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2743 (unsigned long long)dev_addr))
2746 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2747 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2749 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2750 pci_name(pdev), start_pfn, last_pfn);
2752 /* clear the whole page */
2753 dma_pte_clear_range(domain, start_pfn, last_pfn);
2755 /* free page tables */
2756 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2758 if (intel_iommu_strict) {
2759 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2760 last_pfn - start_pfn + 1, 0);
2762 __free_iova(&domain->iovad, iova);
2764 add_unmap(domain, iova);
2766 * queue up the release of the unmap to save the 1/6th of the
2767 * cpu used up by the iotlb flush operation...
2772 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2773 dma_addr_t *dma_handle, gfp_t flags)
2778 size = PAGE_ALIGN(size);
2779 order = get_order(size);
2781 if (!iommu_no_mapping(hwdev))
2782 flags &= ~(GFP_DMA | GFP_DMA32);
2783 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2784 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2790 vaddr = (void *)__get_free_pages(flags, order);
2793 memset(vaddr, 0, size);
2795 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2797 hwdev->coherent_dma_mask);
2800 free_pages((unsigned long)vaddr, order);
2804 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2805 dma_addr_t dma_handle)
2809 size = PAGE_ALIGN(size);
2810 order = get_order(size);
2812 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2813 free_pages((unsigned long)vaddr, order);
2816 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2817 int nelems, enum dma_data_direction dir,
2818 struct dma_attrs *attrs)
2820 struct pci_dev *pdev = to_pci_dev(hwdev);
2821 struct dmar_domain *domain;
2822 unsigned long start_pfn, last_pfn;
2824 struct intel_iommu *iommu;
2826 if (iommu_no_mapping(hwdev))
2829 domain = find_domain(pdev);
2832 iommu = domain_get_iommu(domain);
2834 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2835 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2836 (unsigned long long)sglist[0].dma_address))
2839 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2840 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2842 /* clear the whole page */
2843 dma_pte_clear_range(domain, start_pfn, last_pfn);
2845 /* free page tables */
2846 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2848 if (intel_iommu_strict) {
2849 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2850 last_pfn - start_pfn + 1, 0);
2852 __free_iova(&domain->iovad, iova);
2854 add_unmap(domain, iova);
2856 * queue up the release of the unmap to save the 1/6th of the
2857 * cpu used up by the iotlb flush operation...
2862 static int intel_nontranslate_map_sg(struct device *hddev,
2863 struct scatterlist *sglist, int nelems, int dir)
2866 struct scatterlist *sg;
2868 for_each_sg(sglist, sg, nelems, i) {
2869 BUG_ON(!sg_page(sg));
2870 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2871 sg->dma_length = sg->length;
2876 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2877 enum dma_data_direction dir, struct dma_attrs *attrs)
2880 struct pci_dev *pdev = to_pci_dev(hwdev);
2881 struct dmar_domain *domain;
2884 struct iova *iova = NULL;
2886 struct scatterlist *sg;
2887 unsigned long start_vpfn;
2888 struct intel_iommu *iommu;
2890 BUG_ON(dir == DMA_NONE);
2891 if (iommu_no_mapping(hwdev))
2892 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2894 domain = get_valid_domain_for_dev(pdev);
2898 iommu = domain_get_iommu(domain);
2900 for_each_sg(sglist, sg, nelems, i)
2901 size += aligned_nrpages(sg->offset, sg->length);
2903 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2906 sglist->dma_length = 0;
2911 * Check if DMAR supports zero-length reads on write only
2914 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2915 !cap_zlr(iommu->cap))
2916 prot |= DMA_PTE_READ;
2917 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2918 prot |= DMA_PTE_WRITE;
2920 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2922 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2923 if (unlikely(ret)) {
2924 /* clear the page */
2925 dma_pte_clear_range(domain, start_vpfn,
2926 start_vpfn + size - 1);
2927 /* free page tables */
2928 dma_pte_free_pagetable(domain, start_vpfn,
2929 start_vpfn + size - 1);
2931 __free_iova(&domain->iovad, iova);
2935 /* it's a non-present to present mapping. Only flush if caching mode */
2936 if (cap_caching_mode(iommu->cap))
2937 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
2939 iommu_flush_write_buffer(iommu);
2944 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2949 struct dma_map_ops intel_dma_ops = {
2950 .alloc_coherent = intel_alloc_coherent,
2951 .free_coherent = intel_free_coherent,
2952 .map_sg = intel_map_sg,
2953 .unmap_sg = intel_unmap_sg,
2954 .map_page = intel_map_page,
2955 .unmap_page = intel_unmap_page,
2956 .mapping_error = intel_mapping_error,
2959 static inline int iommu_domain_cache_init(void)
2963 iommu_domain_cache = kmem_cache_create("iommu_domain",
2964 sizeof(struct dmar_domain),
2969 if (!iommu_domain_cache) {
2970 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2977 static inline int iommu_devinfo_cache_init(void)
2981 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2982 sizeof(struct device_domain_info),
2986 if (!iommu_devinfo_cache) {
2987 printk(KERN_ERR "Couldn't create devinfo cache\n");
2994 static inline int iommu_iova_cache_init(void)
2998 iommu_iova_cache = kmem_cache_create("iommu_iova",
2999 sizeof(struct iova),
3003 if (!iommu_iova_cache) {
3004 printk(KERN_ERR "Couldn't create iova cache\n");
3011 static int __init iommu_init_mempool(void)
3014 ret = iommu_iova_cache_init();
3018 ret = iommu_domain_cache_init();
3022 ret = iommu_devinfo_cache_init();
3026 kmem_cache_destroy(iommu_domain_cache);
3028 kmem_cache_destroy(iommu_iova_cache);
3033 static void __init iommu_exit_mempool(void)
3035 kmem_cache_destroy(iommu_devinfo_cache);
3036 kmem_cache_destroy(iommu_domain_cache);
3037 kmem_cache_destroy(iommu_iova_cache);
3041 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3043 struct dmar_drhd_unit *drhd;
3047 /* We know that this device on this chipset has its own IOMMU.
3048 * If we find it under a different IOMMU, then the BIOS is lying
3049 * to us. Hope that the IOMMU for this device is actually
3050 * disabled, and it needs no translation...
3052 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3054 /* "can't" happen */
3055 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3058 vtbar &= 0xffff0000;
3060 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3061 drhd = dmar_find_matched_drhd_unit(pdev);
3062 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3063 TAINT_FIRMWARE_WORKAROUND,
3064 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3065 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3067 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3069 static void __init init_no_remapping_devices(void)
3071 struct dmar_drhd_unit *drhd;
3073 for_each_drhd_unit(drhd) {
3074 if (!drhd->include_all) {
3076 for (i = 0; i < drhd->devices_cnt; i++)
3077 if (drhd->devices[i] != NULL)
3079 /* ignore DMAR unit if no pci devices exist */
3080 if (i == drhd->devices_cnt)
3088 for_each_drhd_unit(drhd) {
3090 if (drhd->ignored || drhd->include_all)
3093 for (i = 0; i < drhd->devices_cnt; i++)
3094 if (drhd->devices[i] &&
3095 !IS_GFX_DEVICE(drhd->devices[i]))
3098 if (i < drhd->devices_cnt)
3101 /* bypass IOMMU if it is just for gfx devices */
3103 for (i = 0; i < drhd->devices_cnt; i++) {
3104 if (!drhd->devices[i])
3106 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3111 #ifdef CONFIG_SUSPEND
3112 static int init_iommu_hw(void)
3114 struct dmar_drhd_unit *drhd;
3115 struct intel_iommu *iommu = NULL;
3117 for_each_active_iommu(iommu, drhd)
3119 dmar_reenable_qi(iommu);
3121 for_each_active_iommu(iommu, drhd) {
3122 iommu_flush_write_buffer(iommu);
3124 iommu_set_root_entry(iommu);
3126 iommu->flush.flush_context(iommu, 0, 0, 0,
3127 DMA_CCMD_GLOBAL_INVL);
3128 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3129 DMA_TLB_GLOBAL_FLUSH);
3130 iommu_enable_translation(iommu);
3131 iommu_disable_protect_mem_regions(iommu);
3137 static void iommu_flush_all(void)
3139 struct dmar_drhd_unit *drhd;
3140 struct intel_iommu *iommu;
3142 for_each_active_iommu(iommu, drhd) {
3143 iommu->flush.flush_context(iommu, 0, 0, 0,
3144 DMA_CCMD_GLOBAL_INVL);
3145 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3146 DMA_TLB_GLOBAL_FLUSH);
3150 static int iommu_suspend(void)
3152 struct dmar_drhd_unit *drhd;
3153 struct intel_iommu *iommu = NULL;
3156 for_each_active_iommu(iommu, drhd) {
3157 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3159 if (!iommu->iommu_state)
3165 for_each_active_iommu(iommu, drhd) {
3166 iommu_disable_translation(iommu);
3168 spin_lock_irqsave(&iommu->register_lock, flag);
3170 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3171 readl(iommu->reg + DMAR_FECTL_REG);
3172 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3173 readl(iommu->reg + DMAR_FEDATA_REG);
3174 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3175 readl(iommu->reg + DMAR_FEADDR_REG);
3176 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3177 readl(iommu->reg + DMAR_FEUADDR_REG);
3179 spin_unlock_irqrestore(&iommu->register_lock, flag);
3184 for_each_active_iommu(iommu, drhd)
3185 kfree(iommu->iommu_state);
3190 static void iommu_resume(void)
3192 struct dmar_drhd_unit *drhd;
3193 struct intel_iommu *iommu = NULL;
3196 if (init_iommu_hw()) {
3197 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3201 for_each_active_iommu(iommu, drhd) {
3203 spin_lock_irqsave(&iommu->register_lock, flag);
3205 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3206 iommu->reg + DMAR_FECTL_REG);
3207 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3208 iommu->reg + DMAR_FEDATA_REG);
3209 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3210 iommu->reg + DMAR_FEADDR_REG);
3211 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3212 iommu->reg + DMAR_FEUADDR_REG);
3214 spin_unlock_irqrestore(&iommu->register_lock, flag);
3217 for_each_active_iommu(iommu, drhd)
3218 kfree(iommu->iommu_state);
3221 static struct syscore_ops iommu_syscore_ops = {
3222 .resume = iommu_resume,
3223 .suspend = iommu_suspend,
3226 static void __init init_iommu_pm_ops(void)
3228 register_syscore_ops(&iommu_syscore_ops);
3232 static inline int init_iommu_pm_ops(void) { }
3233 #endif /* CONFIG_PM */
3236 * Here we only respond to action of unbound device from driver.
3238 * Added device is not attached to its DMAR domain here yet. That will happen
3239 * when mapping the device to iova.
3241 static int device_notifier(struct notifier_block *nb,
3242 unsigned long action, void *data)
3244 struct device *dev = data;
3245 struct pci_dev *pdev = to_pci_dev(dev);
3246 struct dmar_domain *domain;
3248 if (iommu_no_mapping(dev))
3251 domain = find_domain(pdev);
3255 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3256 domain_remove_one_dev_info(domain, pdev);
3258 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3259 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3260 list_empty(&domain->devices))
3261 domain_exit(domain);
3267 static struct notifier_block device_nb = {
3268 .notifier_call = device_notifier,
3271 int __init intel_iommu_init(void)
3276 /* VT-d is required for a TXT/tboot launch, so enforce that */
3277 force_on = tboot_force_iommu();
3279 if (dmar_table_init()) {
3281 panic("tboot: Failed to initialize DMAR table\n");
3285 if (dmar_dev_scope_init()) {
3287 panic("tboot: Failed to initialize DMAR device scope\n");
3292 * Check the need for DMA-remapping initialization now.
3293 * Above initialization will also be used by Interrupt-remapping.
3295 if (no_iommu || dmar_disabled)
3298 if (iommu_init_mempool()) {
3300 panic("tboot: Failed to initialize iommu memory\n");
3304 if (dmar_init_reserved_ranges()) {
3306 panic("tboot: Failed to reserve iommu ranges\n");
3310 init_no_remapping_devices();
3312 ret = init_dmars(force_on);
3315 panic("tboot: Failed to initialize DMARs\n");
3316 printk(KERN_ERR "IOMMU: dmar init failed\n");
3317 put_iova_domain(&reserved_iova_list);
3318 iommu_exit_mempool();
3322 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3324 init_timer(&unmap_timer);
3325 #ifdef CONFIG_SWIOTLB
3328 dma_ops = &intel_dma_ops;
3330 init_iommu_pm_ops();
3332 register_iommu(&intel_iommu_ops);
3334 bus_register_notifier(&pci_bus_type, &device_nb);
3339 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3340 struct pci_dev *pdev)
3342 struct pci_dev *tmp, *parent;
3344 if (!iommu || !pdev)
3347 /* dependent device detach */
3348 tmp = pci_find_upstream_pcie_bridge(pdev);
3349 /* Secondary interface's bus number and devfn 0 */
3351 parent = pdev->bus->self;
3352 while (parent != tmp) {
3353 iommu_detach_dev(iommu, parent->bus->number,
3355 parent = parent->bus->self;
3357 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3358 iommu_detach_dev(iommu,
3359 tmp->subordinate->number, 0);
3360 else /* this is a legacy PCI bridge */
3361 iommu_detach_dev(iommu, tmp->bus->number,
3366 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3367 struct pci_dev *pdev)
3369 struct device_domain_info *info;
3370 struct intel_iommu *iommu;
3371 unsigned long flags;
3373 struct list_head *entry, *tmp;
3375 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3380 spin_lock_irqsave(&device_domain_lock, flags);
3381 list_for_each_safe(entry, tmp, &domain->devices) {
3382 info = list_entry(entry, struct device_domain_info, link);
3383 /* No need to compare PCI domain; it has to be the same */
3384 if (info->bus == pdev->bus->number &&
3385 info->devfn == pdev->devfn) {
3386 list_del(&info->link);
3387 list_del(&info->global);
3389 info->dev->dev.archdata.iommu = NULL;
3390 spin_unlock_irqrestore(&device_domain_lock, flags);
3392 iommu_disable_dev_iotlb(info);
3393 iommu_detach_dev(iommu, info->bus, info->devfn);
3394 iommu_detach_dependent_devices(iommu, pdev);
3395 free_devinfo_mem(info);
3397 spin_lock_irqsave(&device_domain_lock, flags);
3405 /* if there is no other devices under the same iommu
3406 * owned by this domain, clear this iommu in iommu_bmp
3407 * update iommu count and coherency
3409 if (iommu == device_to_iommu(info->segment, info->bus,
3415 unsigned long tmp_flags;
3416 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3417 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3418 domain->iommu_count--;
3419 domain_update_iommu_cap(domain);
3420 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3422 spin_lock_irqsave(&iommu->lock, tmp_flags);
3423 clear_bit(domain->id, iommu->domain_ids);
3424 iommu->domains[domain->id] = NULL;
3425 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3428 spin_unlock_irqrestore(&device_domain_lock, flags);
3431 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3433 struct device_domain_info *info;
3434 struct intel_iommu *iommu;
3435 unsigned long flags1, flags2;
3437 spin_lock_irqsave(&device_domain_lock, flags1);
3438 while (!list_empty(&domain->devices)) {
3439 info = list_entry(domain->devices.next,
3440 struct device_domain_info, link);
3441 list_del(&info->link);
3442 list_del(&info->global);
3444 info->dev->dev.archdata.iommu = NULL;
3446 spin_unlock_irqrestore(&device_domain_lock, flags1);
3448 iommu_disable_dev_iotlb(info);
3449 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3450 iommu_detach_dev(iommu, info->bus, info->devfn);
3451 iommu_detach_dependent_devices(iommu, info->dev);
3453 /* clear this iommu in iommu_bmp, update iommu count
3456 spin_lock_irqsave(&domain->iommu_lock, flags2);
3457 if (test_and_clear_bit(iommu->seq_id,
3458 &domain->iommu_bmp)) {
3459 domain->iommu_count--;
3460 domain_update_iommu_cap(domain);
3462 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3464 free_devinfo_mem(info);
3465 spin_lock_irqsave(&device_domain_lock, flags1);
3467 spin_unlock_irqrestore(&device_domain_lock, flags1);
3470 /* domain id for virtual machine, it won't be set in context */
3471 static unsigned long vm_domid;
3473 static struct dmar_domain *iommu_alloc_vm_domain(void)
3475 struct dmar_domain *domain;
3477 domain = alloc_domain_mem();
3481 domain->id = vm_domid++;
3483 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3484 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3489 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3493 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3494 spin_lock_init(&domain->iommu_lock);
3496 domain_reserve_special_ranges(domain);
3498 /* calculate AGAW */
3499 domain->gaw = guest_width;
3500 adjust_width = guestwidth_to_adjustwidth(guest_width);
3501 domain->agaw = width_to_agaw(adjust_width);
3503 INIT_LIST_HEAD(&domain->devices);
3505 domain->iommu_count = 0;
3506 domain->iommu_coherency = 0;
3507 domain->iommu_snooping = 0;
3508 domain->max_addr = 0;
3511 /* always allocate the top pgd */
3512 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3515 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3519 static void iommu_free_vm_domain(struct dmar_domain *domain)
3521 unsigned long flags;
3522 struct dmar_drhd_unit *drhd;
3523 struct intel_iommu *iommu;
3525 unsigned long ndomains;
3527 for_each_drhd_unit(drhd) {
3530 iommu = drhd->iommu;
3532 ndomains = cap_ndoms(iommu->cap);
3533 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3534 if (iommu->domains[i] == domain) {
3535 spin_lock_irqsave(&iommu->lock, flags);
3536 clear_bit(i, iommu->domain_ids);
3537 iommu->domains[i] = NULL;
3538 spin_unlock_irqrestore(&iommu->lock, flags);
3545 static void vm_domain_exit(struct dmar_domain *domain)
3547 /* Domain 0 is reserved, so dont process it */
3551 vm_domain_remove_all_dev_info(domain);
3553 put_iova_domain(&domain->iovad);
3556 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3558 /* free page tables */
3559 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3561 iommu_free_vm_domain(domain);
3562 free_domain_mem(domain);
3565 static int intel_iommu_domain_init(struct iommu_domain *domain)
3567 struct dmar_domain *dmar_domain;
3569 dmar_domain = iommu_alloc_vm_domain();
3572 "intel_iommu_domain_init: dmar_domain == NULL\n");
3575 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3577 "intel_iommu_domain_init() failed\n");
3578 vm_domain_exit(dmar_domain);
3581 domain->priv = dmar_domain;
3586 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3588 struct dmar_domain *dmar_domain = domain->priv;
3590 domain->priv = NULL;
3591 vm_domain_exit(dmar_domain);
3594 static int intel_iommu_attach_device(struct iommu_domain *domain,
3597 struct dmar_domain *dmar_domain = domain->priv;
3598 struct pci_dev *pdev = to_pci_dev(dev);
3599 struct intel_iommu *iommu;
3602 /* normally pdev is not mapped */
3603 if (unlikely(domain_context_mapped(pdev))) {
3604 struct dmar_domain *old_domain;
3606 old_domain = find_domain(pdev);
3608 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3609 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3610 domain_remove_one_dev_info(old_domain, pdev);
3612 domain_remove_dev_info(old_domain);
3616 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3621 /* check if this iommu agaw is sufficient for max mapped address */
3622 addr_width = agaw_to_width(iommu->agaw);
3623 if (addr_width > cap_mgaw(iommu->cap))
3624 addr_width = cap_mgaw(iommu->cap);
3626 if (dmar_domain->max_addr > (1LL << addr_width)) {
3627 printk(KERN_ERR "%s: iommu width (%d) is not "
3628 "sufficient for the mapped address (%llx)\n",
3629 __func__, addr_width, dmar_domain->max_addr);
3632 dmar_domain->gaw = addr_width;
3635 * Knock out extra levels of page tables if necessary
3637 while (iommu->agaw < dmar_domain->agaw) {
3638 struct dma_pte *pte;
3640 pte = dmar_domain->pgd;
3641 if (dma_pte_present(pte)) {
3642 dmar_domain->pgd = (struct dma_pte *)
3643 phys_to_virt(dma_pte_addr(pte));
3644 free_pgtable_page(pte);
3646 dmar_domain->agaw--;
3649 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3652 static void intel_iommu_detach_device(struct iommu_domain *domain,
3655 struct dmar_domain *dmar_domain = domain->priv;
3656 struct pci_dev *pdev = to_pci_dev(dev);
3658 domain_remove_one_dev_info(dmar_domain, pdev);
3661 static int intel_iommu_map(struct iommu_domain *domain,
3662 unsigned long iova, phys_addr_t hpa,
3663 int gfp_order, int iommu_prot)
3665 struct dmar_domain *dmar_domain = domain->priv;
3671 if (iommu_prot & IOMMU_READ)
3672 prot |= DMA_PTE_READ;
3673 if (iommu_prot & IOMMU_WRITE)
3674 prot |= DMA_PTE_WRITE;
3675 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3676 prot |= DMA_PTE_SNP;
3678 size = PAGE_SIZE << gfp_order;
3679 max_addr = iova + size;
3680 if (dmar_domain->max_addr < max_addr) {
3683 /* check if minimum agaw is sufficient for mapped address */
3684 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3685 if (end < max_addr) {
3686 printk(KERN_ERR "%s: iommu width (%d) is not "
3687 "sufficient for the mapped address (%llx)\n",
3688 __func__, dmar_domain->gaw, max_addr);
3691 dmar_domain->max_addr = max_addr;
3693 /* Round up size to next multiple of PAGE_SIZE, if it and
3694 the low bits of hpa would take us onto the next page */
3695 size = aligned_nrpages(hpa, size);
3696 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3697 hpa >> VTD_PAGE_SHIFT, size, prot);
3701 static int intel_iommu_unmap(struct iommu_domain *domain,
3702 unsigned long iova, int gfp_order)
3704 struct dmar_domain *dmar_domain = domain->priv;
3705 size_t size = PAGE_SIZE << gfp_order;
3707 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3708 (iova + size - 1) >> VTD_PAGE_SHIFT);
3710 if (dmar_domain->max_addr == iova + size)
3711 dmar_domain->max_addr = iova;
3716 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3719 struct dmar_domain *dmar_domain = domain->priv;
3720 struct dma_pte *pte;
3723 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3725 phys = dma_pte_addr(pte);
3730 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3733 struct dmar_domain *dmar_domain = domain->priv;
3735 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3736 return dmar_domain->iommu_snooping;
3737 if (cap == IOMMU_CAP_INTR_REMAP)
3738 return intr_remapping_enabled;
3743 static struct iommu_ops intel_iommu_ops = {
3744 .domain_init = intel_iommu_domain_init,
3745 .domain_destroy = intel_iommu_domain_destroy,
3746 .attach_dev = intel_iommu_attach_device,
3747 .detach_dev = intel_iommu_detach_device,
3748 .map = intel_iommu_map,
3749 .unmap = intel_iommu_unmap,
3750 .iova_to_phys = intel_iommu_iova_to_phys,
3751 .domain_has_cap = intel_iommu_domain_has_cap,
3754 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3757 * Mobile 4 Series Chipset neglects to set RWBF capability,
3760 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3763 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3764 if (dev->revision == 0x07) {
3765 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3770 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3773 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
3774 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3775 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
3776 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
3777 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3778 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3779 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3780 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3782 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3786 if (pci_read_config_word(dev, GGC, &ggc))
3789 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3790 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3794 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3795 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3796 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3797 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3799 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3800 ISOCH DMAR unit for the Azalia sound device, but not give it any
3801 TLB entries, which causes it to deadlock. Check for that. We do
3802 this in a function called from init_dmars(), instead of in a PCI
3803 quirk, because we don't want to print the obnoxious "BIOS broken"
3804 message if VT-d is actually disabled.
3806 static void __init check_tylersburg_isoch(void)
3808 struct pci_dev *pdev;
3809 uint32_t vtisochctrl;
3811 /* If there's no Azalia in the system anyway, forget it. */
3812 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3817 /* System Management Registers. Might be hidden, in which case
3818 we can't do the sanity check. But that's OK, because the
3819 known-broken BIOSes _don't_ actually hide it, so far. */
3820 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3824 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3831 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3832 if (vtisochctrl & 1)
3835 /* Drop all bits other than the number of TLB entries */
3836 vtisochctrl &= 0x1c;
3838 /* If we have the recommended number of TLB entries (16), fine. */
3839 if (vtisochctrl == 0x10)
3842 /* Zero TLB entries? You get to ride the short bus to school. */
3844 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3845 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3846 dmi_get_system_info(DMI_BIOS_VENDOR),
3847 dmi_get_system_info(DMI_BIOS_VERSION),
3848 dmi_get_system_info(DMI_PRODUCT_VERSION));
3849 iommu_identity_mapping |= IDENTMAP_AZALIA;
3853 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",