arch/powerpc/kvm/book3s_64_mmu_hv.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  16  */
  17
  18 #include <linux/types.h>
  19 #include <linux/string.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/highmem.h>
  23 #include <linux/gfp.h>
  24 #include <linux/slab.h>
  25 #include <linux/hugetlb.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/srcu.h>
  28 #include <linux/anon_inodes.h>
  29 #include <linux/file.h>
  30 #include <linux/debugfs.h>
  31
  32 #include <asm/tlbflush.h>
  33 #include <asm/kvm_ppc.h>
  34 #include <asm/kvm_book3s.h>
  35 #include <asm/mmu-hash64.h>
  36 #include <asm/hvcall.h>
  37 #include <asm/synch.h>
  38 #include <asm/ppc-opcode.h>
  39 #include <asm/cputable.h>
  40
  41 #include "trace_hv.h"
  42
  43 /* Power architecture requires HPT is at least 256kB */
  44 #define PPC_MIN_HPT_ORDER       18
  45
  46 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
  47                                 long pte_index, unsigned long pteh,
  48                                 unsigned long ptel, unsigned long *pte_idx_ret);
  49 static void kvmppc_rmap_reset(struct kvm *kvm);
  50
  51 long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp)
  52 {
  53         unsigned long hpt = 0;
  54         struct revmap_entry *rev;
  55         struct page *page = NULL;
  56         long order = KVM_DEFAULT_HPT_ORDER;
  57
  58         if (htab_orderp) {
  59                 order = *htab_orderp;
  60                 if (order < PPC_MIN_HPT_ORDER)
  61                         order = PPC_MIN_HPT_ORDER;
  62         }
  63
  64         kvm->arch.hpt_cma_alloc = 0;
  65         page = kvm_alloc_hpt(1ul << (order - PAGE_SHIFT));
  66         if (page) {
  67                 hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
  68                 memset((void *)hpt, 0, (1ul << order));
  69                 kvm->arch.hpt_cma_alloc = 1;
  70         }
  71
  72         /* Lastly try successively smaller sizes from the page allocator */
  73         while (!hpt && order > PPC_MIN_HPT_ORDER) {
  74                 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_REPEAT|
  75                                        __GFP_NOWARN, order - PAGE_SHIFT);
  76                 if (!hpt)
  77                         --order;
  78         }
  79
  80         if (!hpt)
  81                 return -ENOMEM;
  82
  83         kvm->arch.hpt_virt = hpt;
  84         kvm->arch.hpt_order = order;
  85         /* HPTEs are 2**4 bytes long */
  86         kvm->arch.hpt_npte = 1ul << (order - 4);
  87         /* 128 (2**7) bytes in each HPTEG */
  88         kvm->arch.hpt_mask = (1ul << (order - 7)) - 1;
  89
  90         /* Allocate reverse map array */
  91         rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte);
  92         if (!rev) {
  93                 pr_err("kvmppc_alloc_hpt: Couldn't alloc reverse map array\n");
  94                 goto out_freehpt;
  95         }
  96         kvm->arch.revmap = rev;
  97         kvm->arch.sdr1 = __pa(hpt) | (order - 18);
  98
  99         pr_info("KVM guest htab at %lx (order %ld), LPID %x\n",
 100                 hpt, order, kvm->arch.lpid);
 101
 102         if (htab_orderp)
 103                 *htab_orderp = order;
 104         return 0;
 105
 106  out_freehpt:
 107         if (kvm->arch.hpt_cma_alloc)
 108                 kvm_release_hpt(page, 1 << (order - PAGE_SHIFT));
 109         else
 110                 free_pages(hpt, order - PAGE_SHIFT);
 111         return -ENOMEM;
 112 }
 113
 114 long kvmppc_alloc_reset_hpt(struct kvm *kvm, u32 *htab_orderp)
 115 {
 116         long err = -EBUSY;
 117         long order;
 118
 119         mutex_lock(&kvm->lock);
 120         if (kvm->arch.hpte_setup_done) {
 121                 kvm->arch.hpte_setup_done = 0;
 122                 /* order hpte_setup_done vs. vcpus_running */
 123                 smp_mb();
 124                 if (atomic_read(&kvm->arch.vcpus_running)) {
 125                         kvm->arch.hpte_setup_done = 1;
 126                         goto out;
 127                 }
 128         }
 129         if (kvm->arch.hpt_virt) {
 130                 order = kvm->arch.hpt_order;
 131                 /* Set the entire HPT to 0, i.e. invalid HPTEs */
 132                 memset((void *)kvm->arch.hpt_virt, 0, 1ul << order);
 133                 /*
 134                  * Reset all the reverse-mapping chains for all memslots
 135                  */
 136                 kvmppc_rmap_reset(kvm);
 137                 /* Ensure that each vcpu will flush its TLB on next entry. */
 138                 cpumask_setall(&kvm->arch.need_tlb_flush);
 139                 *htab_orderp = order;
 140                 err = 0;
 141         } else {
 142                 err = kvmppc_alloc_hpt(kvm, htab_orderp);
 143                 order = *htab_orderp;
 144         }
 145  out:
 146         mutex_unlock(&kvm->lock);
 147         return err;
 148 }
 149
 150 void kvmppc_free_hpt(struct kvm *kvm)
 151 {
 152         kvmppc_free_lpid(kvm->arch.lpid);
 153         vfree(kvm->arch.revmap);
 154         if (kvm->arch.hpt_cma_alloc)
 155                 kvm_release_hpt(virt_to_page(kvm->arch.hpt_virt),
 156                                 1 << (kvm->arch.hpt_order - PAGE_SHIFT));
 157         else
 158                 free_pages(kvm->arch.hpt_virt,
 159                            kvm->arch.hpt_order - PAGE_SHIFT);
 160 }
 161
 162 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
 163 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
 164 {
 165         return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
 166 }
 167
 168 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
 169 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
 170 {
 171         return (pgsize == 0x10000) ? 0x1000 : 0;
 172 }
 173
 174 void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 175                      unsigned long porder)
 176 {
 177         unsigned long i;
 178         unsigned long npages;
 179         unsigned long hp_v, hp_r;
 180         unsigned long addr, hash;
 181         unsigned long psize;
 182         unsigned long hp0, hp1;
 183         unsigned long idx_ret;
 184         long ret;
 185         struct kvm *kvm = vcpu->kvm;
 186
 187         psize = 1ul << porder;
 188         npages = memslot->npages >> (porder - PAGE_SHIFT);
 189
 190         /* VRMA can't be > 1TB */
 191         if (npages > 1ul << (40 - porder))
 192                 npages = 1ul << (40 - porder);
 193         /* Can't use more than 1 HPTE per HPTEG */
 194         if (npages > kvm->arch.hpt_mask + 1)
 195                 npages = kvm->arch.hpt_mask + 1;
 196
 197         hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
 198                 HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
 199         hp1 = hpte1_pgsize_encoding(psize) |
 200                 HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;
 201
 202         for (i = 0; i < npages; ++i) {
 203                 addr = i << porder;
 204                 /* can't use hpt_hash since va > 64 bits */
 205                 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25))) & kvm->arch.hpt_mask;
 206                 /*
 207                  * We assume that the hash table is empty and no
 208                  * vcpus are using it at this stage.  Since we create
 209                  * at most one HPTE per HPTEG, we just assume entry 7
 210                  * is available and use it.
 211                  */
 212                 hash = (hash << 3) + 7;
 213                 hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
 214                 hp_r = hp1 | addr;
 215                 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
 216                                                  &idx_ret);
 217                 if (ret != H_SUCCESS) {
 218                         pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
 219                                addr, ret);
 220                         break;
 221                 }
 222         }
 223 }
 224
 225 int kvmppc_mmu_hv_init(void)
 226 {
 227         unsigned long host_lpid, rsvd_lpid;
 228
 229         if (!cpu_has_feature(CPU_FTR_HVMODE))
 230                 return -EINVAL;
 231
 232         /* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
 233         host_lpid = mfspr(SPRN_LPID);
 234         rsvd_lpid = LPID_RSVD;
 235
 236         kvmppc_init_lpid(rsvd_lpid + 1);
 237
 238         kvmppc_claim_lpid(host_lpid);
 239         /* rsvd_lpid is reserved for use in partition switching */
 240         kvmppc_claim_lpid(rsvd_lpid);
 241
 242         return 0;
 243 }
 244
 245 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
 246 {
 247         unsigned long msr = vcpu->arch.intr_msr;
 248
 249         /* If transactional, change to suspend mode on IRQ delivery */
 250         if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
 251                 msr |= MSR_TS_S;
 252         else
 253                 msr |= vcpu->arch.shregs.msr & MSR_TS_MASK;
 254         kvmppc_set_msr(vcpu, msr);
 255 }
 256
 257 long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 258                                 long pte_index, unsigned long pteh,
 259                                 unsigned long ptel, unsigned long *pte_idx_ret)
 260 {
 261         long ret;
 262
 263         /* Protect linux PTE lookup from page table destruction */
 264         rcu_read_lock_sched();  /* this disables preemption too */
 265         ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
 266                                 current->mm->pgd, false, pte_idx_ret);
 267         rcu_read_unlock_sched();
 268         if (ret == H_TOO_HARD) {
 269                 /* this can't happen */
 270                 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
 271                 ret = H_RESOURCE;       /* or something */
 272         }
 273         return ret;
 274
 275 }
 276
 277 static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
 278                                                          gva_t eaddr)
 279 {
 280         u64 mask;
 281         int i;
 282
 283         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 284                 if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
 285                         continue;
 286
 287                 if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
 288                         mask = ESID_MASK_1T;
 289                 else
 290                         mask = ESID_MASK;
 291
 292                 if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
 293                         return &vcpu->arch.slb[i];
 294         }
 295         return NULL;
 296 }
 297
 298 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
 299                         unsigned long ea)
 300 {
 301         unsigned long ra_mask;
 302
 303         ra_mask = hpte_page_size(v, r) - 1;
 304         return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
 305 }
 306
 307 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 308                         struct kvmppc_pte *gpte, bool data, bool iswrite)
 309 {
 310         struct kvm *kvm = vcpu->kvm;
 311         struct kvmppc_slb *slbe;
 312         unsigned long slb_v;
 313         unsigned long pp, key;
 314         unsigned long v, gr;
 315         __be64 *hptep;
 316         int index;
 317         int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
 318
 319         /* Get SLB entry */
 320         if (virtmode) {
 321                 slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
 322                 if (!slbe)
 323                         return -EINVAL;
 324                 slb_v = slbe->origv;
 325         } else {
 326                 /* real mode access */
 327                 slb_v = vcpu->kvm->arch.vrma_slb_v;
 328         }
 329
 330         preempt_disable();
 331         /* Find the HPTE in the hash table */
 332         index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
 333                                          HPTE_V_VALID | HPTE_V_ABSENT);
 334         if (index < 0) {
 335                 preempt_enable();
 336                 return -ENOENT;
 337         }
 338         hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
 339         v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 340         gr = kvm->arch.revmap[index].guest_rpte;
 341
 342         unlock_hpte(hptep, v);
 343         preempt_enable();
 344
 345         gpte->eaddr = eaddr;
 346         gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);
 347
 348         /* Get PP bits and key for permission check */
 349         pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
 350         key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
 351         key &= slb_v;
 352
 353         /* Calculate permissions */
 354         gpte->may_read = hpte_read_permission(pp, key);
 355         gpte->may_write = hpte_write_permission(pp, key);
 356         gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));
 357
 358         /* Storage key permission check for POWER7 */
 359         if (data && virtmode) {
 360                 int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
 361                 if (amrfield & 1)
 362                         gpte->may_read = 0;
 363                 if (amrfield & 2)
 364                         gpte->may_write = 0;
 365         }
 366
 367         /* Get the guest physical address */
 368         gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
 369         return 0;
 370 }
 371
 372 /*
 373  * Quick test for whether an instruction is a load or a store.
 374  * If the instruction is a load or a store, then this will indicate
 375  * which it is, at least on server processors.  (Embedded processors
 376  * have some external PID instructions that don't follow the rule
 377  * embodied here.)  If the instruction isn't a load or store, then
 378  * this doesn't return anything useful.
 379  */
 380 static int instruction_is_store(unsigned int instr)
 381 {
 382         unsigned int mask;
 383
 384         mask = 0x10000000;
 385         if ((instr & 0xfc000000) == 0x7c000000)
 386                 mask = 0x100;           /* major opcode 31 */
 387         return (instr & mask) != 0;
 388 }
 389
 390 static int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu,
 391                                   unsigned long gpa, gva_t ea, int is_store)
 392 {
 393         u32 last_inst;
 394
 395         /*
 396          * If we fail, we just return to the guest and try executing it again.
 397          */
 398         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
 399                 EMULATE_DONE)
 400                 return RESUME_GUEST;
 401
 402         /*
 403          * WARNING: We do not know for sure whether the instruction we just
 404          * read from memory is the same that caused the fault in the first
 405          * place.  If the instruction we read is neither an load or a store,
 406          * then it can't access memory, so we don't need to worry about
 407          * enforcing access permissions.  So, assuming it is a load or
 408          * store, we just check that its direction (load or store) is
 409          * consistent with the original fault, since that's what we
 410          * checked the access permissions against.  If there is a mismatch
 411          * we just return and retry the instruction.
 412          */
 413
 414         if (instruction_is_store(last_inst) != !!is_store)
 415                 return RESUME_GUEST;
 416
 417         /*
 418          * Emulated accesses are emulated by looking at the hash for
 419          * translation once, then performing the access later. The
 420          * translation could be invalidated in the meantime in which
 421          * point performing the subsequent memory access on the old
 422          * physical address could possibly be a security hole for the
 423          * guest (but not the host).
 424          *
 425          * This is less of an issue for MMIO stores since they aren't
 426          * globally visible. It could be an issue for MMIO loads to
 427          * a certain extent but we'll ignore it for now.
 428          */
 429
 430         vcpu->arch.paddr_accessed = gpa;
 431         vcpu->arch.vaddr_accessed = ea;
 432         return kvmppc_emulate_mmio(run, vcpu);
 433 }
 434
 435 int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 436                                 unsigned long ea, unsigned long dsisr)
 437 {
 438         struct kvm *kvm = vcpu->kvm;
 439         unsigned long hpte[3], r;
 440         __be64 *hptep;
 441         unsigned long mmu_seq, psize, pte_size;
 442         unsigned long gpa_base, gfn_base;
 443         unsigned long gpa, gfn, hva, pfn;
 444         struct kvm_memory_slot *memslot;
 445         unsigned long *rmap;
 446         struct revmap_entry *rev;
 447         struct page *page, *pages[1];
 448         long index, ret, npages;
 449         unsigned long is_io;
 450         unsigned int writing, write_ok;
 451         struct vm_area_struct *vma;
 452         unsigned long rcbits;
 453
 454         /*
 455          * Real-mode code has already searched the HPT and found the
 456          * entry we're interested in.  Lock the entry and check that
 457          * it hasn't changed.  If it has, just return and re-execute the
 458          * instruction.
 459          */
 460         if (ea != vcpu->arch.pgfault_addr)
 461                 return RESUME_GUEST;
 462         index = vcpu->arch.pgfault_index;
 463         hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
 464         rev = &kvm->arch.revmap[index];
 465         preempt_disable();
 466         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 467                 cpu_relax();
 468         hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 469         hpte[1] = be64_to_cpu(hptep[1]);
 470         hpte[2] = r = rev->guest_rpte;
 471         unlock_hpte(hptep, hpte[0]);
 472         preempt_enable();
 473
 474         if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 475             hpte[1] != vcpu->arch.pgfault_hpte[1])
 476                 return RESUME_GUEST;
 477
 478         /* Translate the logical address and get the page */
 479         psize = hpte_page_size(hpte[0], r);
 480         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 481         gfn_base = gpa_base >> PAGE_SHIFT;
 482         gpa = gpa_base | (ea & (psize - 1));
 483         gfn = gpa >> PAGE_SHIFT;
 484         memslot = gfn_to_memslot(kvm, gfn);
 485
 486         trace_kvm_page_fault_enter(vcpu, hpte, memslot, ea, dsisr);
 487
 488         /* No memslot means it's an emulated MMIO region */
 489         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 490                 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 491                                               dsisr & DSISR_ISSTORE);
 492
 493         /*
 494          * This should never happen, because of the slot_is_aligned()
 495          * check in kvmppc_do_h_enter().
 496          */
 497         if (gfn_base < memslot->base_gfn)
 498                 return -EFAULT;
 499
 500         /* used to check for invalidations in progress */
 501         mmu_seq = kvm->mmu_notifier_seq;
 502         smp_rmb();
 503
 504         ret = -EFAULT;
 505         is_io = 0;
 506         pfn = 0;
 507         page = NULL;
 508         pte_size = PAGE_SIZE;
 509         writing = (dsisr & DSISR_ISSTORE) != 0;
 510         /* If writing != 0, then the HPTE must allow writing, if we get here */
 511         write_ok = writing;
 512         hva = gfn_to_hva_memslot(memslot, gfn);
 513         npages = get_user_pages_fast(hva, 1, writing, pages);
 514         if (npages < 1) {
 515                 /* Check if it's an I/O mapping */
 516                 down_read(&current->mm->mmap_sem);
 517                 vma = find_vma(current->mm, hva);
 518                 if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end &&
 519                     (vma->vm_flags & VM_PFNMAP)) {
 520                         pfn = vma->vm_pgoff +
 521                                 ((hva - vma->vm_start) >> PAGE_SHIFT);
 522                         pte_size = psize;
 523                         is_io = hpte_cache_bits(pgprot_val(vma->vm_page_prot));
 524                         write_ok = vma->vm_flags & VM_WRITE;
 525                 }
 526                 up_read(&current->mm->mmap_sem);
 527                 if (!pfn)
 528                         goto out_put;
 529         } else {
 530                 page = pages[0];
 531                 pfn = page_to_pfn(page);
 532                 if (PageHuge(page)) {
 533                         page = compound_head(page);
 534                         pte_size <<= compound_order(page);
 535                 }
 536                 /* if the guest wants write access, see if that is OK */
 537                 if (!writing && hpte_is_writable(r)) {
 538                         unsigned int hugepage_shift;
 539                         pte_t *ptep, pte;
 540
 541                         /*
 542                          * We need to protect against page table destruction
 543                          * while looking up and updating the pte.
 544                          */
 545                         rcu_read_lock_sched();
 546                         ptep = find_linux_pte_or_hugepte(current->mm->pgd,
 547                                                          hva, &hugepage_shift);
 548                         if (ptep) {
 549                                 pte = kvmppc_read_update_linux_pte(ptep, 1,
 550                                                            hugepage_shift);
 551                                 if (pte_write(pte))
 552                                         write_ok = 1;
 553                         }
 554                         rcu_read_unlock_sched();
 555                 }
 556         }
 557
 558         if (psize > pte_size)
 559                 goto out_put;
 560
 561         /* Check WIMG vs. the actual page we're accessing */
 562         if (!hpte_cache_flags_ok(r, is_io)) {
 563                 if (is_io)
 564                         goto out_put;
 565
 566                 /*
 567                  * Allow guest to map emulated device memory as
 568                  * uncacheable, but actually make it cacheable.
 569                  */
 570                 r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
 571         }
 572
 573         /*
 574          * Set the HPTE to point to pfn.
 575          * Since the pfn is at PAGE_SIZE granularity, make sure we
 576          * don't mask out lower-order bits if psize < PAGE_SIZE.
 577          */
 578         if (psize < PAGE_SIZE)
 579                 psize = PAGE_SIZE;
 580         r = (r & ~(HPTE_R_PP0 - psize)) | ((pfn << PAGE_SHIFT) & ~(psize - 1));
 581         if (hpte_is_writable(r) && !write_ok)
 582                 r = hpte_make_readonly(r);
 583         ret = RESUME_GUEST;
 584         preempt_disable();
 585         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 586                 cpu_relax();
 587         if ((be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK) != hpte[0] ||
 588                 be64_to_cpu(hptep[1]) != hpte[1] ||
 589                 rev->guest_rpte != hpte[2])
 590                 /* HPTE has been changed under us; let the guest retry */
 591                 goto out_unlock;
 592         hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 593
 594         /* Always put the HPTE in the rmap chain for the page base address */
 595         rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
 596         lock_rmap(rmap);
 597
 598         /* Check if we might have been invalidated; let the guest retry if so */
 599         ret = RESUME_GUEST;
 600         if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
 601                 unlock_rmap(rmap);
 602                 goto out_unlock;
 603         }
 604
 605         /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
 606         rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
 607         r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
 608
 609         if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
 610                 /* HPTE was previously valid, so we need to invalidate it */
 611                 unlock_rmap(rmap);
 612                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 613                 kvmppc_invalidate_hpte(kvm, hptep, index);
 614                 /* don't lose previous R and C bits */
 615                 r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 616         } else {
 617                 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 618         }
 619
 620         hptep[1] = cpu_to_be64(r);
 621         eieio();
 622         __unlock_hpte(hptep, hpte[0]);
 623         asm volatile("ptesync" : : : "memory");
 624         preempt_enable();
 625         if (page && hpte_is_writable(r))
 626                 SetPageDirty(page);
 627
 628  out_put:
 629         trace_kvm_page_fault_exit(vcpu, hpte, ret);
 630
 631         if (page) {
 632                 /*
 633                  * We drop pages[0] here, not page because page might
 634                  * have been set to the head page of a compound, but
 635                  * we have to drop the reference on the correct tail
 636                  * page to match the get inside gup()
 637                  */
 638                 put_page(pages[0]);
 639         }
 640         return ret;
 641
 642  out_unlock:
 643         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 644         preempt_enable();
 645         goto out_put;
 646 }
 647
 648 static void kvmppc_rmap_reset(struct kvm *kvm)
 649 {
 650         struct kvm_memslots *slots;
 651         struct kvm_memory_slot *memslot;
 652         int srcu_idx;
 653
 654         srcu_idx = srcu_read_lock(&kvm->srcu);
 655         slots = kvm->memslots;
 656         kvm_for_each_memslot(memslot, slots) {
 657                 /*
 658                  * This assumes it is acceptable to lose reference and
 659                  * change bits across a reset.
 660                  */
 661                 memset(memslot->arch.rmap, 0,
 662                        memslot->npages * sizeof(*memslot->arch.rmap));
 663         }
 664         srcu_read_unlock(&kvm->srcu, srcu_idx);
 665 }
 666
 667 static int kvm_handle_hva_range(struct kvm *kvm,
 668                                 unsigned long start,
 669                                 unsigned long end,
 670                                 int (*handler)(struct kvm *kvm,
 671                                                unsigned long *rmapp,
 672                                                unsigned long gfn))
 673 {
 674         int ret;
 675         int retval = 0;
 676         struct kvm_memslots *slots;
 677         struct kvm_memory_slot *memslot;
 678
 679         slots = kvm_memslots(kvm);
 680         kvm_for_each_memslot(memslot, slots) {
 681                 unsigned long hva_start, hva_end;
 682                 gfn_t gfn, gfn_end;
 683
 684                 hva_start = max(start, memslot->userspace_addr);
 685                 hva_end = min(end, memslot->userspace_addr +
 686                                         (memslot->npages << PAGE_SHIFT));
 687                 if (hva_start >= hva_end)
 688                         continue;
 689                 /*
 690                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
 691                  * {gfn, gfn+1, ..., gfn_end-1}.
 692                  */
 693                 gfn = hva_to_gfn_memslot(hva_start, memslot);
 694                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 695
 696                 for (; gfn < gfn_end; ++gfn) {
 697                         gfn_t gfn_offset = gfn - memslot->base_gfn;
 698
 699                         ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn);
 700                         retval |= ret;
 701                 }
 702         }
 703
 704         return retval;
 705 }
 706
 707 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 708                           int (*handler)(struct kvm *kvm, unsigned long *rmapp,
 709                                          unsigned long gfn))
 710 {
 711         return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
 712 }
 713
 714 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 715                            unsigned long gfn)
 716 {
 717         struct revmap_entry *rev = kvm->arch.revmap;
 718         unsigned long h, i, j;
 719         __be64 *hptep;
 720         unsigned long ptel, psize, rcbits;
 721
 722         for (;;) {
 723                 lock_rmap(rmapp);
 724                 if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 725                         unlock_rmap(rmapp);
 726                         break;
 727                 }
 728
 729                 /*
 730                  * To avoid an ABBA deadlock with the HPTE lock bit,
 731                  * we can't spin on the HPTE lock while holding the
 732                  * rmap chain lock.
 733                  */
 734                 i = *rmapp & KVMPPC_RMAP_INDEX;
 735                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
 736                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 737                         /* unlock rmap before spinning on the HPTE lock */
 738                         unlock_rmap(rmapp);
 739                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 740                                 cpu_relax();
 741                         continue;
 742                 }
 743                 j = rev[i].forw;
 744                 if (j == i) {
 745                         /* chain is now empty */
 746                         *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
 747                 } else {
 748                         /* remove i from chain */
 749                         h = rev[i].back;
 750                         rev[h].forw = j;
 751                         rev[j].back = h;
 752                         rev[i].forw = rev[i].back = i;
 753                         *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
 754                 }
 755
 756                 /* Now check and modify the HPTE */
 757                 ptel = rev[i].guest_rpte;
 758                 psize = hpte_page_size(be64_to_cpu(hptep[0]), ptel);
 759                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 760                     hpte_rpn(ptel, psize) == gfn) {
 761                         hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 762                         kvmppc_invalidate_hpte(kvm, hptep, i);
 763                         /* Harvest R and C */
 764                         rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 765                         *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
 766                         if (rcbits & ~rev[i].guest_rpte) {
 767                                 rev[i].guest_rpte = ptel | rcbits;
 768                                 note_hpte_modification(kvm, &rev[i]);
 769                         }
 770                 }
 771                 unlock_rmap(rmapp);
 772                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 773         }
 774         return 0;
 775 }
 776
 777 int kvm_unmap_hva_hv(struct kvm *kvm, unsigned long hva)
 778 {
 779         kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
 780         return 0;
 781 }
 782
 783 int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 784 {
 785         kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp);
 786         return 0;
 787 }
 788
 789 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
 790                                   struct kvm_memory_slot *memslot)
 791 {
 792         unsigned long *rmapp;
 793         unsigned long gfn;
 794         unsigned long n;
 795
 796         rmapp = memslot->arch.rmap;
 797         gfn = memslot->base_gfn;
 798         for (n = memslot->npages; n; --n) {
 799                 /*
 800                  * Testing the present bit without locking is OK because
 801                  * the memslot has been marked invalid already, and hence
 802                  * no new HPTEs referencing this page can be created,
 803                  * thus the present bit can't go from 0 to 1.
 804                  */
 805                 if (*rmapp & KVMPPC_RMAP_PRESENT)
 806                         kvm_unmap_rmapp(kvm, rmapp, gfn);
 807                 ++rmapp;
 808                 ++gfn;
 809         }
 810 }
 811
 812 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 813                          unsigned long gfn)
 814 {
 815         struct revmap_entry *rev = kvm->arch.revmap;
 816         unsigned long head, i, j;
 817         __be64 *hptep;
 818         int ret = 0;
 819
 820  retry:
 821         lock_rmap(rmapp);
 822         if (*rmapp & KVMPPC_RMAP_REFERENCED) {
 823                 *rmapp &= ~KVMPPC_RMAP_REFERENCED;
 824                 ret = 1;
 825         }
 826         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 827                 unlock_rmap(rmapp);
 828                 return ret;
 829         }
 830
 831         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 832         do {
 833                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
 834                 j = rev[i].forw;
 835
 836                 /* If this HPTE isn't referenced, ignore it */
 837                 if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
 838                         continue;
 839
 840                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 841                         /* unlock rmap before spinning on the HPTE lock */
 842                         unlock_rmap(rmapp);
 843                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 844                                 cpu_relax();
 845                         goto retry;
 846                 }
 847
 848                 /* Now check and modify the HPTE */
 849                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 850                     (be64_to_cpu(hptep[1]) & HPTE_R_R)) {
 851                         kvmppc_clear_ref_hpte(kvm, hptep, i);
 852                         if (!(rev[i].guest_rpte & HPTE_R_R)) {
 853                                 rev[i].guest_rpte |= HPTE_R_R;
 854                                 note_hpte_modification(kvm, &rev[i]);
 855                         }
 856                         ret = 1;
 857                 }
 858                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 859         } while ((i = j) != head);
 860
 861         unlock_rmap(rmapp);
 862         return ret;
 863 }
 864
 865 int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 866 {
 867         return kvm_handle_hva_range(kvm, start, end, kvm_age_rmapp);
 868 }
 869
 870 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 871                               unsigned long gfn)
 872 {
 873         struct revmap_entry *rev = kvm->arch.revmap;
 874         unsigned long head, i, j;
 875         unsigned long *hp;
 876         int ret = 1;
 877
 878         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 879                 return 1;
 880
 881         lock_rmap(rmapp);
 882         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 883                 goto out;
 884
 885         if (*rmapp & KVMPPC_RMAP_PRESENT) {
 886                 i = head = *rmapp & KVMPPC_RMAP_INDEX;
 887                 do {
 888                         hp = (unsigned long *)(kvm->arch.hpt_virt + (i << 4));
 889                         j = rev[i].forw;
 890                         if (be64_to_cpu(hp[1]) & HPTE_R_R)
 891                                 goto out;
 892                 } while ((i = j) != head);
 893         }
 894         ret = 0;
 895
 896  out:
 897         unlock_rmap(rmapp);
 898         return ret;
 899 }
 900
 901 int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)
 902 {
 903         return kvm_handle_hva(kvm, hva, kvm_test_age_rmapp);
 904 }
 905
 906 void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)
 907 {
 908         kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
 909 }
 910
 911 static int vcpus_running(struct kvm *kvm)
 912 {
 913         return atomic_read(&kvm->arch.vcpus_running) != 0;
 914 }
 915
 916 /*
 917  * Returns the number of system pages that are dirty.
 918  * This can be more than 1 if we find a huge-page HPTE.
 919  */
 920 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
 921 {
 922         struct revmap_entry *rev = kvm->arch.revmap;
 923         unsigned long head, i, j;
 924         unsigned long n;
 925         unsigned long v, r;
 926         __be64 *hptep;
 927         int npages_dirty = 0;
 928
 929  retry:
 930         lock_rmap(rmapp);
 931         if (*rmapp & KVMPPC_RMAP_CHANGED) {
 932                 *rmapp &= ~KVMPPC_RMAP_CHANGED;
 933                 npages_dirty = 1;
 934         }
 935         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 936                 unlock_rmap(rmapp);
 937                 return npages_dirty;
 938         }
 939
 940         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 941         do {
 942                 unsigned long hptep1;
 943                 hptep = (__be64 *) (kvm->arch.hpt_virt + (i << 4));
 944                 j = rev[i].forw;
 945
 946                 /*
 947                  * Checking the C (changed) bit here is racy since there
 948                  * is no guarantee about when the hardware writes it back.
 949                  * If the HPTE is not writable then it is stable since the
 950                  * page can't be written to, and we would have done a tlbie
 951                  * (which forces the hardware to complete any writeback)
 952                  * when making the HPTE read-only.
 953                  * If vcpus are running then this call is racy anyway
 954                  * since the page could get dirtied subsequently, so we
 955                  * expect there to be a further call which would pick up
 956                  * any delayed C bit writeback.
 957                  * Otherwise we need to do the tlbie even if C==0 in
 958                  * order to pick up any delayed writeback of C.
 959                  */
 960                 hptep1 = be64_to_cpu(hptep[1]);
 961                 if (!(hptep1 & HPTE_R_C) &&
 962                     (!hpte_is_writable(hptep1) || vcpus_running(kvm)))
 963                         continue;
 964
 965                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 966                         /* unlock rmap before spinning on the HPTE lock */
 967                         unlock_rmap(rmapp);
 968                         while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
 969                                 cpu_relax();
 970                         goto retry;
 971                 }
 972
 973                 /* Now check and modify the HPTE */
 974                 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
 975                         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 976                         continue;
 977                 }
 978
 979                 /* need to make it temporarily absent so C is stable */
 980                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 981                 kvmppc_invalidate_hpte(kvm, hptep, i);
 982                 v = be64_to_cpu(hptep[0]);
 983                 r = be64_to_cpu(hptep[1]);
 984                 if (r & HPTE_R_C) {
 985                         hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
 986                         if (!(rev[i].guest_rpte & HPTE_R_C)) {
 987                                 rev[i].guest_rpte |= HPTE_R_C;
 988                                 note_hpte_modification(kvm, &rev[i]);
 989                         }
 990                         n = hpte_page_size(v, r);
 991                         n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
 992                         if (n > npages_dirty)
 993                                 npages_dirty = n;
 994                         eieio();
 995                 }
 996                 v &= ~HPTE_V_ABSENT;
 997                 v |= HPTE_V_VALID;
 998                 __unlock_hpte(hptep, v);
 999         } while ((i = j) != head);
1000
1001         unlock_rmap(rmapp);
1002         return npages_dirty;
1003 }
1004
1005 static void harvest_vpa_dirty(struct kvmppc_vpa *vpa,
1006                               struct kvm_memory_slot *memslot,
1007                               unsigned long *map)
1008 {
1009         unsigned long gfn;
1010
1011         if (!vpa->dirty || !vpa->pinned_addr)
1012                 return;
1013         gfn = vpa->gpa >> PAGE_SHIFT;
1014         if (gfn < memslot->base_gfn ||
1015             gfn >= memslot->base_gfn + memslot->npages)
1016                 return;
1017
1018         vpa->dirty = false;
1019         if (map)
1020                 __set_bit_le(gfn - memslot->base_gfn, map);
1021 }
1022
1023 long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,
1024                              unsigned long *map)
1025 {
1026         unsigned long i, j;
1027         unsigned long *rmapp;
1028         struct kvm_vcpu *vcpu;
1029
1030         preempt_disable();
1031         rmapp = memslot->arch.rmap;
1032         for (i = 0; i < memslot->npages; ++i) {
1033                 int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
1034                 /*
1035                  * Note that if npages > 0 then i must be a multiple of npages,
1036                  * since we always put huge-page HPTEs in the rmap chain
1037                  * corresponding to their page base address.
1038                  */
1039                 if (npages && map)
1040                         for (j = i; npages; ++j, --npages)
1041                                 __set_bit_le(j, map);
1042                 ++rmapp;
1043         }
1044
1045         /* Harvest dirty bits from VPA and DTL updates */
1046         /* Note: we never modify the SLB shadow buffer areas */
1047         kvm_for_each_vcpu(i, vcpu, kvm) {
1048                 spin_lock(&vcpu->arch.vpa_update_lock);
1049                 harvest_vpa_dirty(&vcpu->arch.vpa, memslot, map);
1050                 harvest_vpa_dirty(&vcpu->arch.dtl, memslot, map);
1051                 spin_unlock(&vcpu->arch.vpa_update_lock);
1052         }
1053         preempt_enable();
1054         return 0;
1055 }
1056
1057 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
1058                             unsigned long *nb_ret)
1059 {
1060         struct kvm_memory_slot *memslot;
1061         unsigned long gfn = gpa >> PAGE_SHIFT;
1062         struct page *page, *pages[1];
1063         int npages;
1064         unsigned long hva, offset;
1065         int srcu_idx;
1066
1067         srcu_idx = srcu_read_lock(&kvm->srcu);
1068         memslot = gfn_to_memslot(kvm, gfn);
1069         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1070                 goto err;
1071         hva = gfn_to_hva_memslot(memslot, gfn);
1072         npages = get_user_pages_fast(hva, 1, 1, pages);
1073         if (npages < 1)
1074                 goto err;
1075         page = pages[0];
1076         srcu_read_unlock(&kvm->srcu, srcu_idx);
1077
1078         offset = gpa & (PAGE_SIZE - 1);
1079         if (nb_ret)
1080                 *nb_ret = PAGE_SIZE - offset;
1081         return page_address(page) + offset;
1082
1083  err:
1084         srcu_read_unlock(&kvm->srcu, srcu_idx);
1085         return NULL;
1086 }
1087
1088 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
1089                              bool dirty)
1090 {
1091         struct page *page = virt_to_page(va);
1092         struct kvm_memory_slot *memslot;
1093         unsigned long gfn;
1094         unsigned long *rmap;
1095         int srcu_idx;
1096
1097         put_page(page);
1098
1099         if (!dirty)
1100                 return;
1101
1102         /* We need to mark this page dirty in the rmap chain */
1103         gfn = gpa >> PAGE_SHIFT;
1104         srcu_idx = srcu_read_lock(&kvm->srcu);
1105         memslot = gfn_to_memslot(kvm, gfn);
1106         if (memslot) {
1107                 rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
1108                 lock_rmap(rmap);
1109                 *rmap |= KVMPPC_RMAP_CHANGED;
1110                 unlock_rmap(rmap);
1111         }
1112         srcu_read_unlock(&kvm->srcu, srcu_idx);
1113 }
1114
1115 /*
1116  * Functions for reading and writing the hash table via reads and
1117  * writes on a file descriptor.
1118  *
1119  * Reads return the guest view of the hash table, which has to be
1120  * pieced together from the real hash table and the guest_rpte
1121  * values in the revmap array.
1122  *
1123  * On writes, each HPTE written is considered in turn, and if it
1124  * is valid, it is written to the HPT as if an H_ENTER with the
1125  * exact flag set was done.  When the invalid count is non-zero
1126  * in the header written to the stream, the kernel will make
1127  * sure that that many HPTEs are invalid, and invalidate them
1128  * if not.
1129  */
1130
1131 struct kvm_htab_ctx {
1132         unsigned long   index;
1133         unsigned long   flags;
1134         struct kvm      *kvm;
1135         int             first_pass;
1136 };
1137
1138 #define HPTE_SIZE       (2 * sizeof(unsigned long))
1139
1140 /*
1141  * Returns 1 if this HPT entry has been modified or has pending
1142  * R/C bit changes.
1143  */
1144 static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
1145 {
1146         unsigned long rcbits_unset;
1147
1148         if (revp->guest_rpte & HPTE_GR_MODIFIED)
1149                 return 1;
1150
1151         /* Also need to consider changes in reference and changed bits */
1152         rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1153         if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
1154             (be64_to_cpu(hptp[1]) & rcbits_unset))
1155                 return 1;
1156
1157         return 0;
1158 }
1159
1160 static long record_hpte(unsigned long flags, __be64 *hptp,
1161                         unsigned long *hpte, struct revmap_entry *revp,
1162                         int want_valid, int first_pass)
1163 {
1164         unsigned long v, r;
1165         unsigned long rcbits_unset;
1166         int ok = 1;
1167         int valid, dirty;
1168
1169         /* Unmodified entries are uninteresting except on the first pass */
1170         dirty = hpte_dirty(revp, hptp);
1171         if (!first_pass && !dirty)
1172                 return 0;
1173
1174         valid = 0;
1175         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1176                 valid = 1;
1177                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
1178                     !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
1179                         valid = 0;
1180         }
1181         if (valid != want_valid)
1182                 return 0;
1183
1184         v = r = 0;
1185         if (valid || dirty) {
1186                 /* lock the HPTE so it's stable and read it */
1187                 preempt_disable();
1188                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1189                         cpu_relax();
1190                 v = be64_to_cpu(hptp[0]);
1191
1192                 /* re-evaluate valid and dirty from synchronized HPTE value */
1193                 valid = !!(v & HPTE_V_VALID);
1194                 dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
1195
1196                 /* Harvest R and C into guest view if necessary */
1197                 rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1198                 if (valid && (rcbits_unset & be64_to_cpu(hptp[1]))) {
1199                         revp->guest_rpte |= (be64_to_cpu(hptp[1]) &
1200                                 (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
1201                         dirty = 1;
1202                 }
1203
1204                 if (v & HPTE_V_ABSENT) {
1205                         v &= ~HPTE_V_ABSENT;
1206                         v |= HPTE_V_VALID;
1207                         valid = 1;
1208                 }
1209                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
1210                         valid = 0;
1211
1212                 r = revp->guest_rpte;
1213                 /* only clear modified if this is the right sort of entry */
1214                 if (valid == want_valid && dirty) {
1215                         r &= ~HPTE_GR_MODIFIED;
1216                         revp->guest_rpte = r;
1217                 }
1218                 unlock_hpte(hptp, be64_to_cpu(hptp[0]));
1219                 preempt_enable();
1220                 if (!(valid == want_valid && (first_pass || dirty)))
1221                         ok = 0;
1222         }
1223         hpte[0] = cpu_to_be64(v);
1224         hpte[1] = cpu_to_be64(r);
1225         return ok;
1226 }
1227
1228 static ssize_t kvm_htab_read(struct file *file, char __user *buf,
1229                              size_t count, loff_t *ppos)
1230 {
1231         struct kvm_htab_ctx *ctx = file->private_data;
1232         struct kvm *kvm = ctx->kvm;
1233         struct kvm_get_htab_header hdr;
1234         __be64 *hptp;
1235         struct revmap_entry *revp;
1236         unsigned long i, nb, nw;
1237         unsigned long __user *lbuf;
1238         struct kvm_get_htab_header __user *hptr;
1239         unsigned long flags;
1240         int first_pass;
1241         unsigned long hpte[2];
1242
1243         if (!access_ok(VERIFY_WRITE, buf, count))
1244                 return -EFAULT;
1245
1246         first_pass = ctx->first_pass;
1247         flags = ctx->flags;
1248
1249         i = ctx->index;
1250         hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1251         revp = kvm->arch.revmap + i;
1252         lbuf = (unsigned long __user *)buf;
1253
1254         nb = 0;
1255         while (nb + sizeof(hdr) + HPTE_SIZE < count) {
1256                 /* Initialize header */
1257                 hptr = (struct kvm_get_htab_header __user *)buf;
1258                 hdr.n_valid = 0;
1259                 hdr.n_invalid = 0;
1260                 nw = nb;
1261                 nb += sizeof(hdr);
1262                 lbuf = (unsigned long __user *)(buf + sizeof(hdr));
1263
1264                 /* Skip uninteresting entries, i.e. clean on not-first pass */
1265                 if (!first_pass) {
1266                         while (i < kvm->arch.hpt_npte &&
1267                                !hpte_dirty(revp, hptp)) {
1268                                 ++i;
1269                                 hptp += 2;
1270                                 ++revp;
1271                         }
1272                 }
1273                 hdr.index = i;
1274
1275                 /* Grab a series of valid entries */
1276                 while (i < kvm->arch.hpt_npte &&
1277                        hdr.n_valid < 0xffff &&
1278                        nb + HPTE_SIZE < count &&
1279                        record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
1280                         /* valid entry, write it out */
1281                         ++hdr.n_valid;
1282                         if (__put_user(hpte[0], lbuf) ||
1283                             __put_user(hpte[1], lbuf + 1))
1284                                 return -EFAULT;
1285                         nb += HPTE_SIZE;
1286                         lbuf += 2;
1287                         ++i;
1288                         hptp += 2;
1289                         ++revp;
1290                 }
1291                 /* Now skip invalid entries while we can */
1292                 while (i < kvm->arch.hpt_npte &&
1293                        hdr.n_invalid < 0xffff &&
1294                        record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
1295                         /* found an invalid entry */
1296                         ++hdr.n_invalid;
1297                         ++i;
1298                         hptp += 2;
1299                         ++revp;
1300                 }
1301
1302                 if (hdr.n_valid || hdr.n_invalid) {
1303                         /* write back the header */
1304                         if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
1305                                 return -EFAULT;
1306                         nw = nb;
1307                         buf = (char __user *)lbuf;
1308                 } else {
1309                         nb = nw;
1310                 }
1311
1312                 /* Check if we've wrapped around the hash table */
1313                 if (i >= kvm->arch.hpt_npte) {
1314                         i = 0;
1315                         ctx->first_pass = 0;
1316                         break;
1317                 }
1318         }
1319
1320         ctx->index = i;
1321
1322         return nb;
1323 }
1324
1325 static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1326                               size_t count, loff_t *ppos)
1327 {
1328         struct kvm_htab_ctx *ctx = file->private_data;
1329         struct kvm *kvm = ctx->kvm;
1330         struct kvm_get_htab_header hdr;
1331         unsigned long i, j;
1332         unsigned long v, r;
1333         unsigned long __user *lbuf;
1334         __be64 *hptp;
1335         unsigned long tmp[2];
1336         ssize_t nb;
1337         long int err, ret;
1338         int hpte_setup;
1339
1340         if (!access_ok(VERIFY_READ, buf, count))
1341                 return -EFAULT;
1342
1343         /* lock out vcpus from running while we're doing this */
1344         mutex_lock(&kvm->lock);
1345         hpte_setup = kvm->arch.hpte_setup_done;
1346         if (hpte_setup) {
1347                 kvm->arch.hpte_setup_done = 0;  /* temporarily */
1348                 /* order hpte_setup_done vs. vcpus_running */
1349                 smp_mb();
1350                 if (atomic_read(&kvm->arch.vcpus_running)) {
1351                         kvm->arch.hpte_setup_done = 1;
1352                         mutex_unlock(&kvm->lock);
1353                         return -EBUSY;
1354                 }
1355         }
1356
1357         err = 0;
1358         for (nb = 0; nb + sizeof(hdr) <= count; ) {
1359                 err = -EFAULT;
1360                 if (__copy_from_user(&hdr, buf, sizeof(hdr)))
1361                         break;
1362
1363                 err = 0;
1364                 if (nb + hdr.n_valid * HPTE_SIZE > count)
1365                         break;
1366
1367                 nb += sizeof(hdr);
1368                 buf += sizeof(hdr);
1369
1370                 err = -EINVAL;
1371                 i = hdr.index;
1372                 if (i >= kvm->arch.hpt_npte ||
1373                     i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte)
1374                         break;
1375
1376                 hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1377                 lbuf = (unsigned long __user *)buf;
1378                 for (j = 0; j < hdr.n_valid; ++j) {
1379                         __be64 hpte_v;
1380                         __be64 hpte_r;
1381
1382                         err = -EFAULT;
1383                         if (__get_user(hpte_v, lbuf) ||
1384                             __get_user(hpte_r, lbuf + 1))
1385                                 goto out;
1386                         v = be64_to_cpu(hpte_v);
1387                         r = be64_to_cpu(hpte_r);
1388                         err = -EINVAL;
1389                         if (!(v & HPTE_V_VALID))
1390                                 goto out;
1391                         lbuf += 2;
1392                         nb += HPTE_SIZE;
1393
1394                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1395                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1396                         err = -EIO;
1397                         ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
1398                                                          tmp);
1399                         if (ret != H_SUCCESS) {
1400                                 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1401                                        "r=%lx\n", ret, i, v, r);
1402                                 goto out;
1403                         }
1404                         if (!hpte_setup && is_vrma_hpte(v)) {
1405                                 unsigned long psize = hpte_base_page_size(v, r);
1406                                 unsigned long senc = slb_pgsize_encoding(psize);
1407                                 unsigned long lpcr;
1408
1409                                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1410                                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1411                                 lpcr = senc << (LPCR_VRMASD_SH - 4);
1412                                 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
1413                                 hpte_setup = 1;
1414                         }
1415                         ++i;
1416                         hptp += 2;
1417                 }
1418
1419                 for (j = 0; j < hdr.n_invalid; ++j) {
1420                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1421                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1422                         ++i;
1423                         hptp += 2;
1424                 }
1425                 err = 0;
1426         }
1427
1428  out:
1429         /* Order HPTE updates vs. hpte_setup_done */
1430         smp_wmb();
1431         kvm->arch.hpte_setup_done = hpte_setup;
1432         mutex_unlock(&kvm->lock);
1433
1434         if (err)
1435                 return err;
1436         return nb;
1437 }
1438
1439 static int kvm_htab_release(struct inode *inode, struct file *filp)
1440 {
1441         struct kvm_htab_ctx *ctx = filp->private_data;
1442
1443         filp->private_data = NULL;
1444         if (!(ctx->flags & KVM_GET_HTAB_WRITE))
1445                 atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
1446         kvm_put_kvm(ctx->kvm);
1447         kfree(ctx);
1448         return 0;
1449 }
1450
1451 static const struct file_operations kvm_htab_fops = {
1452         .read           = kvm_htab_read,
1453         .write          = kvm_htab_write,
1454         .llseek         = default_llseek,
1455         .release        = kvm_htab_release,
1456 };
1457
1458 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1459 {
1460         int ret;
1461         struct kvm_htab_ctx *ctx;
1462         int rwflag;
1463
1464         /* reject flags we don't recognize */
1465         if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
1466                 return -EINVAL;
1467         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1468         if (!ctx)
1469                 return -ENOMEM;
1470         kvm_get_kvm(kvm);
1471         ctx->kvm = kvm;
1472         ctx->index = ghf->start_index;
1473         ctx->flags = ghf->flags;
1474         ctx->first_pass = 1;
1475
1476         rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1477         ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1478         if (ret < 0) {
1479                 kvm_put_kvm(kvm);
1480                 return ret;
1481         }
1482
1483         if (rwflag == O_RDONLY) {
1484                 mutex_lock(&kvm->slots_lock);
1485                 atomic_inc(&kvm->arch.hpte_mod_interest);
1486                 /* make sure kvmppc_do_h_enter etc. see the increment */
1487                 synchronize_srcu_expedited(&kvm->srcu);
1488                 mutex_unlock(&kvm->slots_lock);
1489         }
1490
1491         return ret;
1492 }
1493
1494 struct debugfs_htab_state {
1495         struct kvm      *kvm;
1496         struct mutex    mutex;
1497         unsigned long   hpt_index;
1498         int             chars_left;
1499         int             buf_index;
1500         char            buf[64];
1501 };
1502
1503 static int debugfs_htab_open(struct inode *inode, struct file *file)
1504 {
1505         struct kvm *kvm = inode->i_private;
1506         struct debugfs_htab_state *p;
1507
1508         p = kzalloc(sizeof(*p), GFP_KERNEL);
1509         if (!p)
1510                 return -ENOMEM;
1511
1512         kvm_get_kvm(kvm);
1513         p->kvm = kvm;
1514         mutex_init(&p->mutex);
1515         file->private_data = p;
1516
1517         return nonseekable_open(inode, file);
1518 }
1519
1520 static int debugfs_htab_release(struct inode *inode, struct file *file)
1521 {
1522         struct debugfs_htab_state *p = file->private_data;
1523
1524         kvm_put_kvm(p->kvm);
1525         kfree(p);
1526         return 0;
1527 }
1528
1529 static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
1530                                  size_t len, loff_t *ppos)
1531 {
1532         struct debugfs_htab_state *p = file->private_data;
1533         ssize_t ret, r;
1534         unsigned long i, n;
1535         unsigned long v, hr, gr;
1536         struct kvm *kvm;
1537         __be64 *hptp;
1538
1539         ret = mutex_lock_interruptible(&p->mutex);
1540         if (ret)
1541                 return ret;
1542
1543         if (p->chars_left) {
1544                 n = p->chars_left;
1545                 if (n > len)
1546                         n = len;
1547                 r = copy_to_user(buf, p->buf + p->buf_index, n);
1548                 n -= r;
1549                 p->chars_left -= n;
1550                 p->buf_index += n;
1551                 buf += n;
1552                 len -= n;
1553                 ret = n;
1554                 if (r) {
1555                         if (!n)
1556                                 ret = -EFAULT;
1557                         goto out;
1558                 }
1559         }
1560
1561         kvm = p->kvm;
1562         i = p->hpt_index;
1563         hptp = (__be64 *)(kvm->arch.hpt_virt + (i * HPTE_SIZE));
1564         for (; len != 0 && i < kvm->arch.hpt_npte; ++i, hptp += 2) {
1565                 if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)))
1566                         continue;
1567
1568                 /* lock the HPTE so it's stable and read it */
1569                 preempt_disable();
1570                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1571                         cpu_relax();
1572                 v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
1573                 hr = be64_to_cpu(hptp[1]);
1574                 gr = kvm->arch.revmap[i].guest_rpte;
1575                 unlock_hpte(hptp, v);
1576                 preempt_enable();
1577
1578                 if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT)))
1579                         continue;
1580
1581                 n = scnprintf(p->buf, sizeof(p->buf),
1582                               "%6lx %.16lx %.16lx %.16lx\n",
1583                               i, v, hr, gr);
1584                 p->chars_left = n;
1585                 if (n > len)
1586                         n = len;
1587                 r = copy_to_user(buf, p->buf, n);
1588                 n -= r;
1589                 p->chars_left -= n;
1590                 p->buf_index = n;
1591                 buf += n;
1592                 len -= n;
1593                 ret += n;
1594                 if (r) {
1595                         if (!ret)
1596                                 ret = -EFAULT;
1597                         goto out;
1598                 }
1599         }
1600         p->hpt_index = i;
1601
1602  out:
1603         mutex_unlock(&p->mutex);
1604         return ret;
1605 }
1606
1607 ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
1608                            size_t len, loff_t *ppos)
1609 {
1610         return -EACCES;
1611 }
1612
1613 static const struct file_operations debugfs_htab_fops = {
1614         .owner   = THIS_MODULE,
1615         .open    = debugfs_htab_open,
1616         .release = debugfs_htab_release,
1617         .read    = debugfs_htab_read,
1618         .write   = debugfs_htab_write,
1619         .llseek  = generic_file_llseek,
1620 };
1621
1622 void kvmppc_mmu_debugfs_init(struct kvm *kvm)
1623 {
1624         kvm->arch.htab_dentry = debugfs_create_file("htab", 0400,
1625                                                     kvm->arch.debugfs_dir, kvm,
1626                                                     &debugfs_htab_fops);
1627 }
1628
1629 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
1630 {
1631         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
1632
1633         vcpu->arch.slb_nr = 32;         /* POWER7/POWER8 */
1634
1635         mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
1636         mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;
1637
1638         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
1639 }