drivers/kvm/mmu.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  *
  11  * Authors:
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *   Avi Kivity   <avi@qumranet.com>
  14  *
  15  * This work is licensed under the terms of the GNU GPL, version 2.  See
  16  * the COPYING file in the top-level directory.
  17  *
  18  */
  19 #include <linux/types.h>
  20 #include <linux/string.h>
  21 #include <asm/page.h>
  22 #include <linux/mm.h>
  23 #include <linux/highmem.h>
  24 #include <linux/module.h>
  25
  26 #include "vmx.h"
  27 #include "kvm.h"
  28
  29 #define pgprintk(x...) do { } while (0)
  30 #define rmap_printk(x...) do { } while (0)
  31
  32 #define ASSERT(x)                                                       \
  33         if (!(x)) {                                                     \
  34                 printk(KERN_WARNING "assertion failed %s:%d: %s\n",     \
  35                        __FILE__, __LINE__, #x);                         \
  36         }
  37
  38 #define PT64_ENT_PER_PAGE 512
  39 #define PT32_ENT_PER_PAGE 1024
  40
  41 #define PT_WRITABLE_SHIFT 1
  42
  43 #define PT_PRESENT_MASK (1ULL << 0)
  44 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
  45 #define PT_USER_MASK (1ULL << 2)
  46 #define PT_PWT_MASK (1ULL << 3)
  47 #define PT_PCD_MASK (1ULL << 4)
  48 #define PT_ACCESSED_MASK (1ULL << 5)
  49 #define PT_DIRTY_MASK (1ULL << 6)
  50 #define PT_PAGE_SIZE_MASK (1ULL << 7)
  51 #define PT_PAT_MASK (1ULL << 7)
  52 #define PT_GLOBAL_MASK (1ULL << 8)
  53 #define PT64_NX_MASK (1ULL << 63)
  54
  55 #define PT_PAT_SHIFT 7
  56 #define PT_DIR_PAT_SHIFT 12
  57 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
  58
  59 #define PT32_DIR_PSE36_SIZE 4
  60 #define PT32_DIR_PSE36_SHIFT 13
  61 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
  62
  63
  64 #define PT32_PTE_COPY_MASK \
  65         (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
  66
  67 #define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
  68
  69 #define PT_FIRST_AVAIL_BITS_SHIFT 9
  70 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  71
  72 #define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  73 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  74
  75 #define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
  76 #define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
  77
  78 #define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
  79 #define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
  80
  81 #define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
  82
  83 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
  84
  85 #define PT64_LEVEL_BITS 9
  86
  87 #define PT64_LEVEL_SHIFT(level) \
  88                 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
  89
  90 #define PT64_LEVEL_MASK(level) \
  91                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
  92
  93 #define PT64_INDEX(address, level)\
  94         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
  95
  96
  97 #define PT32_LEVEL_BITS 10
  98
  99 #define PT32_LEVEL_SHIFT(level) \
 100                 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
 101
 102 #define PT32_LEVEL_MASK(level) \
 103                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
 104
 105 #define PT32_INDEX(address, level)\
 106         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 107
 108
 109 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
 110 #define PT64_DIR_BASE_ADDR_MASK \
 111         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 112
 113 #define PT32_BASE_ADDR_MASK PAGE_MASK
 114 #define PT32_DIR_BASE_ADDR_MASK \
 115         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 116
 117
 118 #define PFERR_PRESENT_MASK (1U << 0)
 119 #define PFERR_WRITE_MASK (1U << 1)
 120 #define PFERR_USER_MASK (1U << 2)
 121
 122 #define PT64_ROOT_LEVEL 4
 123 #define PT32_ROOT_LEVEL 2
 124 #define PT32E_ROOT_LEVEL 3
 125
 126 #define PT_DIRECTORY_LEVEL 2
 127 #define PT_PAGE_TABLE_LEVEL 1
 128
 129 #define RMAP_EXT 4
 130
 131 struct kvm_rmap_desc {
 132         u64 *shadow_ptes[RMAP_EXT];
 133         struct kvm_rmap_desc *more;
 134 };
 135
 136 static int is_write_protection(struct kvm_vcpu *vcpu)
 137 {
 138         return vcpu->cr0 & CR0_WP_MASK;
 139 }
 140
 141 static int is_cpuid_PSE36(void)
 142 {
 143         return 1;
 144 }
 145
 146 static int is_present_pte(unsigned long pte)
 147 {
 148         return pte & PT_PRESENT_MASK;
 149 }
 150
 151 static int is_writeble_pte(unsigned long pte)
 152 {
 153         return pte & PT_WRITABLE_MASK;
 154 }
 155
 156 static int is_io_pte(unsigned long pte)
 157 {
 158         return pte & PT_SHADOW_IO_MARK;
 159 }
 160
 161 static int is_rmap_pte(u64 pte)
 162 {
 163         return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
 164                 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
 165 }
 166
 167 /*
 168  * Reverse mapping data structures:
 169  *
 170  * If page->private bit zero is zero, then page->private points to the
 171  * shadow page table entry that points to page_address(page).
 172  *
 173  * If page->private bit zero is one, (then page->private & ~1) points
 174  * to a struct kvm_rmap_desc containing more mappings.
 175  */
 176 static void rmap_add(struct kvm *kvm, u64 *spte)
 177 {
 178         struct page *page;
 179         struct kvm_rmap_desc *desc;
 180         int i;
 181
 182         if (!is_rmap_pte(*spte))
 183                 return;
 184         page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
 185         if (!page->private) {
 186                 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
 187                 page->private = (unsigned long)spte;
 188         } else if (!(page->private & 1)) {
 189                 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
 190                 desc = kzalloc(sizeof *desc, GFP_NOWAIT);
 191                 if (!desc)
 192                         BUG(); /* FIXME: return error */
 193                 desc->shadow_ptes[0] = (u64 *)page->private;
 194                 desc->shadow_ptes[1] = spte;
 195                 page->private = (unsigned long)desc | 1;
 196         } else {
 197                 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
 198                 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
 199                 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
 200                         desc = desc->more;
 201                 if (desc->shadow_ptes[RMAP_EXT-1]) {
 202                         desc->more = kzalloc(sizeof *desc->more, GFP_NOWAIT);
 203                         if (!desc->more)
 204                                 BUG(); /* FIXME: return error */
 205                         desc = desc->more;
 206                 }
 207                 for (i = 0; desc->shadow_ptes[i]; ++i)
 208                         ;
 209                 desc->shadow_ptes[i] = spte;
 210         }
 211 }
 212
 213 static void rmap_desc_remove_entry(struct page *page,
 214                                    struct kvm_rmap_desc *desc,
 215                                    int i,
 216                                    struct kvm_rmap_desc *prev_desc)
 217 {
 218         int j;
 219
 220         for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
 221                 ;
 222         desc->shadow_ptes[i] = desc->shadow_ptes[j];
 223         desc->shadow_ptes[j] = 0;
 224         if (j != 0)
 225                 return;
 226         if (!prev_desc && !desc->more)
 227                 page->private = (unsigned long)desc->shadow_ptes[0];
 228         else
 229                 if (prev_desc)
 230                         prev_desc->more = desc->more;
 231                 else
 232                         page->private = (unsigned long)desc->more | 1;
 233         kfree(desc);
 234 }
 235
 236 static void rmap_remove(struct kvm *kvm, u64 *spte)
 237 {
 238         struct page *page;
 239         struct kvm_rmap_desc *desc;
 240         struct kvm_rmap_desc *prev_desc;
 241         int i;
 242
 243         if (!is_rmap_pte(*spte))
 244                 return;
 245         page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
 246         if (!page->private) {
 247                 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
 248                 BUG();
 249         } else if (!(page->private & 1)) {
 250                 rmap_printk("rmap_remove:  %p %llx 1->0\n", spte, *spte);
 251                 if ((u64 *)page->private != spte) {
 252                         printk(KERN_ERR "rmap_remove:  %p %llx 1->BUG\n",
 253                                spte, *spte);
 254                         BUG();
 255                 }
 256                 page->private = 0;
 257         } else {
 258                 rmap_printk("rmap_remove:  %p %llx many->many\n", spte, *spte);
 259                 desc = (struct kvm_rmap_desc *)(page->private & ~1ul);
 260                 prev_desc = NULL;
 261                 while (desc) {
 262                         for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
 263                                 if (desc->shadow_ptes[i] == spte) {
 264                                         rmap_desc_remove_entry(page, desc, i,
 265                                                                prev_desc);
 266                                         return;
 267                                 }
 268                         prev_desc = desc;
 269                         desc = desc->more;
 270                 }
 271                 BUG();
 272         }
 273 }
 274
 275 static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
 276 {
 277         struct kvm_mmu_page *page_head = page_header(page_hpa);
 278
 279         list_del(&page_head->link);
 280         page_head->page_hpa = page_hpa;
 281         list_add(&page_head->link, &vcpu->free_pages);
 282 }
 283
 284 static int is_empty_shadow_page(hpa_t page_hpa)
 285 {
 286         u32 *pos;
 287         u32 *end;
 288         for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32);
 289                       pos != end; pos++)
 290                 if (*pos != 0)
 291                         return 0;
 292         return 1;
 293 }
 294
 295 static hpa_t kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte)
 296 {
 297         struct kvm_mmu_page *page;
 298
 299         if (list_empty(&vcpu->free_pages))
 300                 return INVALID_PAGE;
 301
 302         page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
 303         list_del(&page->link);
 304         list_add(&page->link, &vcpu->kvm->active_mmu_pages);
 305         ASSERT(is_empty_shadow_page(page->page_hpa));
 306         page->slot_bitmap = 0;
 307         page->global = 1;
 308         page->parent_pte = parent_pte;
 309         return page->page_hpa;
 310 }
 311
 312 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
 313 {
 314         int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
 315         struct kvm_mmu_page *page_head = page_header(__pa(pte));
 316
 317         __set_bit(slot, &page_head->slot_bitmap);
 318 }
 319
 320 hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 321 {
 322         hpa_t hpa = gpa_to_hpa(vcpu, gpa);
 323
 324         return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
 325 }
 326
 327 hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 328 {
 329         struct kvm_memory_slot *slot;
 330         struct page *page;
 331
 332         ASSERT((gpa & HPA_ERR_MASK) == 0);
 333         slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
 334         if (!slot)
 335                 return gpa | HPA_ERR_MASK;
 336         page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
 337         return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
 338                 | (gpa & (PAGE_SIZE-1));
 339 }
 340
 341 hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
 342 {
 343         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
 344
 345         if (gpa == UNMAPPED_GVA)
 346                 return UNMAPPED_GVA;
 347         return gpa_to_hpa(vcpu, gpa);
 348 }
 349
 350
 351 static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa,
 352                                int level)
 353 {
 354         u64 *pos;
 355         u64 *end;
 356
 357         ASSERT(vcpu);
 358         ASSERT(VALID_PAGE(page_hpa));
 359         ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
 360
 361         for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE;
 362              pos != end; pos++) {
 363                 u64 current_ent = *pos;
 364
 365                 if (is_present_pte(current_ent)) {
 366                         if (level != 1)
 367                                 release_pt_page_64(vcpu,
 368                                                   current_ent &
 369                                                   PT64_BASE_ADDR_MASK,
 370                                                   level - 1);
 371                         else
 372                                 rmap_remove(vcpu->kvm, pos);
 373                 }
 374                 *pos = 0;
 375         }
 376         kvm_mmu_free_page(vcpu, page_hpa);
 377 }
 378
 379 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
 380 {
 381 }
 382
 383 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
 384 {
 385         int level = PT32E_ROOT_LEVEL;
 386         hpa_t table_addr = vcpu->mmu.root_hpa;
 387
 388         for (; ; level--) {
 389                 u32 index = PT64_INDEX(v, level);
 390                 u64 *table;
 391
 392                 ASSERT(VALID_PAGE(table_addr));
 393                 table = __va(table_addr);
 394
 395                 if (level == 1) {
 396                         mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
 397                         page_header_update_slot(vcpu->kvm, table, v);
 398                         table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
 399                                                                 PT_USER_MASK;
 400                         rmap_add(vcpu->kvm, &table[index]);
 401                         return 0;
 402                 }
 403
 404                 if (table[index] == 0) {
 405                         hpa_t new_table = kvm_mmu_alloc_page(vcpu,
 406                                                              &table[index]);
 407
 408                         if (!VALID_PAGE(new_table)) {
 409                                 pgprintk("nonpaging_map: ENOMEM\n");
 410                                 return -ENOMEM;
 411                         }
 412
 413                         if (level == PT32E_ROOT_LEVEL)
 414                                 table[index] = new_table | PT_PRESENT_MASK;
 415                         else
 416                                 table[index] = new_table | PT_PRESENT_MASK |
 417                                                 PT_WRITABLE_MASK | PT_USER_MASK;
 418                 }
 419                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
 420         }
 421 }
 422
 423 static void mmu_free_roots(struct kvm_vcpu *vcpu)
 424 {
 425         int i;
 426
 427 #ifdef CONFIG_X86_64
 428         if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 429                 hpa_t root = vcpu->mmu.root_hpa;
 430
 431                 ASSERT(VALID_PAGE(root));
 432                 release_pt_page_64(vcpu, root, PT64_ROOT_LEVEL);
 433                 vcpu->mmu.root_hpa = INVALID_PAGE;
 434                 return;
 435         }
 436 #endif
 437         for (i = 0; i < 4; ++i) {
 438                 hpa_t root = vcpu->mmu.pae_root[i];
 439
 440                 ASSERT(VALID_PAGE(root));
 441                 root &= PT64_BASE_ADDR_MASK;
 442                 release_pt_page_64(vcpu, root, PT32E_ROOT_LEVEL - 1);
 443                 vcpu->mmu.pae_root[i] = INVALID_PAGE;
 444         }
 445         vcpu->mmu.root_hpa = INVALID_PAGE;
 446 }
 447
 448 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
 449 {
 450         int i;
 451
 452 #ifdef CONFIG_X86_64
 453         if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
 454                 hpa_t root = vcpu->mmu.root_hpa;
 455
 456                 ASSERT(!VALID_PAGE(root));
 457                 root = kvm_mmu_alloc_page(vcpu, NULL);
 458                 vcpu->mmu.root_hpa = root;
 459                 return;
 460         }
 461 #endif
 462         for (i = 0; i < 4; ++i) {
 463                 hpa_t root = vcpu->mmu.pae_root[i];
 464
 465                 ASSERT(!VALID_PAGE(root));
 466                 root = kvm_mmu_alloc_page(vcpu, NULL);
 467                 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
 468         }
 469         vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
 470 }
 471
 472 static void nonpaging_flush(struct kvm_vcpu *vcpu)
 473 {
 474         hpa_t root = vcpu->mmu.root_hpa;
 475
 476         ++kvm_stat.tlb_flush;
 477         pgprintk("nonpaging_flush\n");
 478         mmu_free_roots(vcpu);
 479         mmu_alloc_roots(vcpu);
 480         kvm_arch_ops->set_cr3(vcpu, root);
 481         kvm_arch_ops->tlb_flush(vcpu);
 482 }
 483
 484 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
 485 {
 486         return vaddr;
 487 }
 488
 489 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
 490                                u32 error_code)
 491 {
 492         int ret;
 493         gpa_t addr = gva;
 494
 495         ASSERT(vcpu);
 496         ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
 497
 498         for (;;) {
 499              hpa_t paddr;
 500
 501              paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
 502
 503              if (is_error_hpa(paddr))
 504                      return 1;
 505
 506              ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
 507              if (ret) {
 508                      nonpaging_flush(vcpu);
 509                      continue;
 510              }
 511              break;
 512         }
 513         return ret;
 514 }
 515
 516 static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
 517 {
 518 }
 519
 520 static void nonpaging_free(struct kvm_vcpu *vcpu)
 521 {
 522         mmu_free_roots(vcpu);
 523 }
 524
 525 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
 526 {
 527         struct kvm_mmu *context = &vcpu->mmu;
 528
 529         context->new_cr3 = nonpaging_new_cr3;
 530         context->page_fault = nonpaging_page_fault;
 531         context->inval_page = nonpaging_inval_page;
 532         context->gva_to_gpa = nonpaging_gva_to_gpa;
 533         context->free = nonpaging_free;
 534         context->root_level = PT32E_ROOT_LEVEL;
 535         context->shadow_root_level = PT32E_ROOT_LEVEL;
 536         mmu_alloc_roots(vcpu);
 537         ASSERT(VALID_PAGE(context->root_hpa));
 538         kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
 539         return 0;
 540 }
 541
 542
 543 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
 544 {
 545         struct kvm_mmu_page *page, *npage;
 546
 547         list_for_each_entry_safe(page, npage, &vcpu->kvm->active_mmu_pages,
 548                                  link) {
 549                 if (page->global)
 550                         continue;
 551
 552                 if (!page->parent_pte)
 553                         continue;
 554
 555                 *page->parent_pte = 0;
 556                 release_pt_page_64(vcpu, page->page_hpa, 1);
 557         }
 558         ++kvm_stat.tlb_flush;
 559         kvm_arch_ops->tlb_flush(vcpu);
 560 }
 561
 562 static void paging_new_cr3(struct kvm_vcpu *vcpu)
 563 {
 564         kvm_mmu_flush_tlb(vcpu);
 565 }
 566
 567 static void mark_pagetable_nonglobal(void *shadow_pte)
 568 {
 569         page_header(__pa(shadow_pte))->global = 0;
 570 }
 571
 572 static inline void set_pte_common(struct kvm_vcpu *vcpu,
 573                              u64 *shadow_pte,
 574                              gpa_t gaddr,
 575                              int dirty,
 576                              u64 access_bits)
 577 {
 578         hpa_t paddr;
 579
 580         *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
 581         if (!dirty)
 582                 access_bits &= ~PT_WRITABLE_MASK;
 583
 584         if (access_bits & PT_WRITABLE_MASK)
 585                 mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
 586
 587         *shadow_pte |= access_bits;
 588
 589         paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
 590
 591         if (!(*shadow_pte & PT_GLOBAL_MASK))
 592                 mark_pagetable_nonglobal(shadow_pte);
 593
 594         if (is_error_hpa(paddr)) {
 595                 *shadow_pte |= gaddr;
 596                 *shadow_pte |= PT_SHADOW_IO_MARK;
 597                 *shadow_pte &= ~PT_PRESENT_MASK;
 598         } else {
 599                 *shadow_pte |= paddr;
 600                 page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
 601                 rmap_add(vcpu->kvm, shadow_pte);
 602         }
 603 }
 604
 605 static void inject_page_fault(struct kvm_vcpu *vcpu,
 606                               u64 addr,
 607                               u32 err_code)
 608 {
 609         kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
 610 }
 611
 612 static inline int fix_read_pf(u64 *shadow_ent)
 613 {
 614         if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
 615             !(*shadow_ent & PT_USER_MASK)) {
 616                 /*
 617                  * If supervisor write protect is disabled, we shadow kernel
 618                  * pages as user pages so we can trap the write access.
 619                  */
 620                 *shadow_ent |= PT_USER_MASK;
 621                 *shadow_ent &= ~PT_WRITABLE_MASK;
 622
 623                 return 1;
 624
 625         }
 626         return 0;
 627 }
 628
 629 static int may_access(u64 pte, int write, int user)
 630 {
 631
 632         if (user && !(pte & PT_USER_MASK))
 633                 return 0;
 634         if (write && !(pte & PT_WRITABLE_MASK))
 635                 return 0;
 636         return 1;
 637 }
 638
 639 /*
 640  * Remove a shadow pte.
 641  */
 642 static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr)
 643 {
 644         hpa_t page_addr = vcpu->mmu.root_hpa;
 645         int level = vcpu->mmu.shadow_root_level;
 646
 647         ++kvm_stat.invlpg;
 648
 649         for (; ; level--) {
 650                 u32 index = PT64_INDEX(addr, level);
 651                 u64 *table = __va(page_addr);
 652
 653                 if (level == PT_PAGE_TABLE_LEVEL ) {
 654                         rmap_remove(vcpu->kvm, &table[index]);
 655                         table[index] = 0;
 656                         return;
 657                 }
 658
 659                 if (!is_present_pte(table[index]))
 660                         return;
 661
 662                 page_addr = table[index] & PT64_BASE_ADDR_MASK;
 663
 664                 if (level == PT_DIRECTORY_LEVEL &&
 665                           (table[index] & PT_SHADOW_PS_MARK)) {
 666                         table[index] = 0;
 667                         release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
 668
 669                         kvm_arch_ops->tlb_flush(vcpu);
 670                         return;
 671                 }
 672         }
 673 }
 674
 675 static void paging_free(struct kvm_vcpu *vcpu)
 676 {
 677         nonpaging_free(vcpu);
 678 }
 679
 680 #define PTTYPE 64
 681 #include "paging_tmpl.h"
 682 #undef PTTYPE
 683
 684 #define PTTYPE 32
 685 #include "paging_tmpl.h"
 686 #undef PTTYPE
 687
 688 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
 689 {
 690         struct kvm_mmu *context = &vcpu->mmu;
 691
 692         ASSERT(is_pae(vcpu));
 693         context->new_cr3 = paging_new_cr3;
 694         context->page_fault = paging64_page_fault;
 695         context->inval_page = paging_inval_page;
 696         context->gva_to_gpa = paging64_gva_to_gpa;
 697         context->free = paging_free;
 698         context->root_level = level;
 699         context->shadow_root_level = level;
 700         mmu_alloc_roots(vcpu);
 701         ASSERT(VALID_PAGE(context->root_hpa));
 702         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
 703                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
 704         return 0;
 705 }
 706
 707 static int paging64_init_context(struct kvm_vcpu *vcpu)
 708 {
 709         return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
 710 }
 711
 712 static int paging32_init_context(struct kvm_vcpu *vcpu)
 713 {
 714         struct kvm_mmu *context = &vcpu->mmu;
 715
 716         context->new_cr3 = paging_new_cr3;
 717         context->page_fault = paging32_page_fault;
 718         context->inval_page = paging_inval_page;
 719         context->gva_to_gpa = paging32_gva_to_gpa;
 720         context->free = paging_free;
 721         context->root_level = PT32_ROOT_LEVEL;
 722         context->shadow_root_level = PT32E_ROOT_LEVEL;
 723         mmu_alloc_roots(vcpu);
 724         ASSERT(VALID_PAGE(context->root_hpa));
 725         kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
 726                     (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
 727         return 0;
 728 }
 729
 730 static int paging32E_init_context(struct kvm_vcpu *vcpu)
 731 {
 732         return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
 733 }
 734
 735 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
 736 {
 737         ASSERT(vcpu);
 738         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
 739
 740         if (!is_paging(vcpu))
 741                 return nonpaging_init_context(vcpu);
 742         else if (is_long_mode(vcpu))
 743                 return paging64_init_context(vcpu);
 744         else if (is_pae(vcpu))
 745                 return paging32E_init_context(vcpu);
 746         else
 747                 return paging32_init_context(vcpu);
 748 }
 749
 750 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
 751 {
 752         ASSERT(vcpu);
 753         if (VALID_PAGE(vcpu->mmu.root_hpa)) {
 754                 vcpu->mmu.free(vcpu);
 755                 vcpu->mmu.root_hpa = INVALID_PAGE;
 756         }
 757 }
 758
 759 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
 760 {
 761         destroy_kvm_mmu(vcpu);
 762         return init_kvm_mmu(vcpu);
 763 }
 764
 765 static void free_mmu_pages(struct kvm_vcpu *vcpu)
 766 {
 767         while (!list_empty(&vcpu->free_pages)) {
 768                 struct kvm_mmu_page *page;
 769
 770                 page = list_entry(vcpu->free_pages.next,
 771                                   struct kvm_mmu_page, link);
 772                 list_del(&page->link);
 773                 __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
 774                 page->page_hpa = INVALID_PAGE;
 775         }
 776         free_page((unsigned long)vcpu->mmu.pae_root);
 777 }
 778
 779 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
 780 {
 781         struct page *page;
 782         int i;
 783
 784         ASSERT(vcpu);
 785
 786         for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
 787                 struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
 788
 789                 INIT_LIST_HEAD(&page_header->link);
 790                 if ((page = alloc_page(GFP_KERNEL)) == NULL)
 791                         goto error_1;
 792                 page->private = (unsigned long)page_header;
 793                 page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
 794                 memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
 795                 list_add(&page_header->link, &vcpu->free_pages);
 796         }
 797
 798         /*
 799          * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
 800          * Therefore we need to allocate shadow page tables in the first
 801          * 4GB of memory, which happens to fit the DMA32 zone.
 802          */
 803         page = alloc_page(GFP_KERNEL | __GFP_DMA32);
 804         if (!page)
 805                 goto error_1;
 806         vcpu->mmu.pae_root = page_address(page);
 807         for (i = 0; i < 4; ++i)
 808                 vcpu->mmu.pae_root[i] = INVALID_PAGE;
 809
 810         return 0;
 811
 812 error_1:
 813         free_mmu_pages(vcpu);
 814         return -ENOMEM;
 815 }
 816
 817 int kvm_mmu_create(struct kvm_vcpu *vcpu)
 818 {
 819         ASSERT(vcpu);
 820         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
 821         ASSERT(list_empty(&vcpu->free_pages));
 822
 823         return alloc_mmu_pages(vcpu);
 824 }
 825
 826 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
 827 {
 828         ASSERT(vcpu);
 829         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
 830         ASSERT(!list_empty(&vcpu->free_pages));
 831
 832         return init_kvm_mmu(vcpu);
 833 }
 834
 835 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
 836 {
 837         ASSERT(vcpu);
 838
 839         destroy_kvm_mmu(vcpu);
 840         free_mmu_pages(vcpu);
 841 }
 842
 843 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
 844 {
 845         struct kvm_mmu_page *page;
 846
 847         list_for_each_entry(page, &kvm->active_mmu_pages, link) {
 848                 int i;
 849                 u64 *pt;
 850
 851                 if (!test_bit(slot, &page->slot_bitmap))
 852                         continue;
 853
 854                 pt = __va(page->page_hpa);
 855                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
 856                         /* avoid RMW */
 857                         if (pt[i] & PT_WRITABLE_MASK) {
 858                                 rmap_remove(kvm, &pt[i]);
 859                                 pt[i] &= ~PT_WRITABLE_MASK;
 860                         }
 861         }
 862 }