arch/x86/kvm/x86.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * derived from drivers/kvm/kvm_main.c
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2008 Qumranet, Inc.
   8  * Copyright IBM Corporation, 2008
   9  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Avi Kivity   <avi@qumranet.com>
  13  *   Yaniv Kamay  <yaniv@qumranet.com>
  14  *   Amit Shah    <amit.shah@qumranet.com>
  15  *   Ben-Ami Yassour <benami@il.ibm.com>
  16  *
  17  * This work is licensed under the terms of the GNU GPL, version 2.  See
  18  * the COPYING file in the top-level directory.
  19  *
  20  */
  21
  22 #include <linux/kvm_host.h>
  23 #include "irq.h"
  24 #include "mmu.h"
  25 #include "i8254.h"
  26 #include "tss.h"
  27 #include "kvm_cache_regs.h"
  28 #include "x86.h"
  29 #include "cpuid.h"
  30
  31 #include <linux/clocksource.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/kvm.h>
  34 #include <linux/fs.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/module.h>
  37 #include <linux/mman.h>
  38 #include <linux/highmem.h>
  39 #include <linux/iommu.h>
  40 #include <linux/intel-iommu.h>
  41 #include <linux/cpufreq.h>
  42 #include <linux/user-return-notifier.h>
  43 #include <linux/srcu.h>
  44 #include <linux/slab.h>
  45 #include <linux/perf_event.h>
  46 #include <linux/uaccess.h>
  47 #include <linux/hash.h>
  48 #include <linux/pci.h>
  49 #include <linux/timekeeper_internal.h>
  50 #include <linux/pvclock_gtod.h>
  51 #include <trace/events/kvm.h>
  52
  53 #define CREATE_TRACE_POINTS
  54 #include "trace.h"
  55
  56 #include <asm/debugreg.h>
  57 #include <asm/msr.h>
  58 #include <asm/desc.h>
  59 #include <asm/mtrr.h>
  60 #include <asm/mce.h>
  61 #include <asm/i387.h>
  62 #include <asm/fpu-internal.h> /* Ugh! */
  63 #include <asm/xcr.h>
  64 #include <asm/pvclock.h>
  65 #include <asm/div64.h>
  66
  67 #define MAX_IO_MSRS 256
  68 #define KVM_MAX_MCE_BANKS 32
  69 #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)
  70
  71 #define emul_to_vcpu(ctxt) \
  72         container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)
  73
  74 /* EFER defaults:
  75  * - enable syscall per default because its emulated by KVM
  76  * - enable LME and LMA per default on 64 bit KVM
  77  */
  78 #ifdef CONFIG_X86_64
  79 static
  80 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
  81 #else
  82 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
  83 #endif
  84
  85 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  86 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  87
  88 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
  89 static void process_nmi(struct kvm_vcpu *vcpu);
  90
  91 struct kvm_x86_ops *kvm_x86_ops;
  92 EXPORT_SYMBOL_GPL(kvm_x86_ops);
  93
  94 static bool ignore_msrs = 0;
  95 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
  96
  97 unsigned int min_timer_period_us = 500;
  98 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
  99
 100 bool kvm_has_tsc_control;
 101 EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
 102 u32  kvm_max_guest_tsc_khz;
 103 EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);
 104
 105 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
 106 static u32 tsc_tolerance_ppm = 250;
 107 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
 108
 109 #define KVM_NR_SHARED_MSRS 16
 110
 111 struct kvm_shared_msrs_global {
 112         int nr;
 113         u32 msrs[KVM_NR_SHARED_MSRS];
 114 };
 115
 116 struct kvm_shared_msrs {
 117         struct user_return_notifier urn;
 118         bool registered;
 119         struct kvm_shared_msr_values {
 120                 u64 host;
 121                 u64 curr;
 122         } values[KVM_NR_SHARED_MSRS];
 123 };
 124
 125 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
 126 static struct kvm_shared_msrs __percpu *shared_msrs;
 127
 128 struct kvm_stats_debugfs_item debugfs_entries[] = {
 129         { "pf_fixed", VCPU_STAT(pf_fixed) },
 130         { "pf_guest", VCPU_STAT(pf_guest) },
 131         { "tlb_flush", VCPU_STAT(tlb_flush) },
 132         { "invlpg", VCPU_STAT(invlpg) },
 133         { "exits", VCPU_STAT(exits) },
 134         { "io_exits", VCPU_STAT(io_exits) },
 135         { "mmio_exits", VCPU_STAT(mmio_exits) },
 136         { "signal_exits", VCPU_STAT(signal_exits) },
 137         { "irq_window", VCPU_STAT(irq_window_exits) },
 138         { "nmi_window", VCPU_STAT(nmi_window_exits) },
 139         { "halt_exits", VCPU_STAT(halt_exits) },
 140         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
 141         { "hypercalls", VCPU_STAT(hypercalls) },
 142         { "request_irq", VCPU_STAT(request_irq_exits) },
 143         { "irq_exits", VCPU_STAT(irq_exits) },
 144         { "host_state_reload", VCPU_STAT(host_state_reload) },
 145         { "efer_reload", VCPU_STAT(efer_reload) },
 146         { "fpu_reload", VCPU_STAT(fpu_reload) },
 147         { "insn_emulation", VCPU_STAT(insn_emulation) },
 148         { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
 149         { "irq_injections", VCPU_STAT(irq_injections) },
 150         { "nmi_injections", VCPU_STAT(nmi_injections) },
 151         { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
 152         { "mmu_pte_write", VM_STAT(mmu_pte_write) },
 153         { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
 154         { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
 155         { "mmu_flooded", VM_STAT(mmu_flooded) },
 156         { "mmu_recycled", VM_STAT(mmu_recycled) },
 157         { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
 158         { "mmu_unsync", VM_STAT(mmu_unsync) },
 159         { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
 160         { "largepages", VM_STAT(lpages) },
 161         { NULL }
 162 };
 163
 164 u64 __read_mostly host_xcr0;
 165
 166 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
 167
 168 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
 169 {
 170         int i;
 171         for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
 172                 vcpu->arch.apf.gfns[i] = ~0;
 173 }
 174
 175 static void kvm_on_user_return(struct user_return_notifier *urn)
 176 {
 177         unsigned slot;
 178         struct kvm_shared_msrs *locals
 179                 = container_of(urn, struct kvm_shared_msrs, urn);
 180         struct kvm_shared_msr_values *values;
 181
 182         for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
 183                 values = &locals->values[slot];
 184                 if (values->host != values->curr) {
 185                         wrmsrl(shared_msrs_global.msrs[slot], values->host);
 186                         values->curr = values->host;
 187                 }
 188         }
 189         locals->registered = false;
 190         user_return_notifier_unregister(urn);
 191 }
 192
 193 static void shared_msr_update(unsigned slot, u32 msr)
 194 {
 195         u64 value;
 196         unsigned int cpu = smp_processor_id();
 197         struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
 198
 199         /* only read, and nobody should modify it at this time,
 200          * so don't need lock */
 201         if (slot >= shared_msrs_global.nr) {
 202                 printk(KERN_ERR "kvm: invalid MSR slot!");
 203                 return;
 204         }
 205         rdmsrl_safe(msr, &value);
 206         smsr->values[slot].host = value;
 207         smsr->values[slot].curr = value;
 208 }
 209
 210 void kvm_define_shared_msr(unsigned slot, u32 msr)
 211 {
 212         if (slot >= shared_msrs_global.nr)
 213                 shared_msrs_global.nr = slot + 1;
 214         shared_msrs_global.msrs[slot] = msr;
 215         /* we need ensured the shared_msr_global have been updated */
 216         smp_wmb();
 217 }
 218 EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
 219
 220 static void kvm_shared_msr_cpu_online(void)
 221 {
 222         unsigned i;
 223
 224         for (i = 0; i < shared_msrs_global.nr; ++i)
 225                 shared_msr_update(i, shared_msrs_global.msrs[i]);
 226 }
 227
 228 void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
 229 {
 230         unsigned int cpu = smp_processor_id();
 231         struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
 232
 233         if (((value ^ smsr->values[slot].curr) & mask) == 0)
 234                 return;
 235         smsr->values[slot].curr = value;
 236         wrmsrl(shared_msrs_global.msrs[slot], value);
 237         if (!smsr->registered) {
 238                 smsr->urn.on_user_return = kvm_on_user_return;
 239                 user_return_notifier_register(&smsr->urn);
 240                 smsr->registered = true;
 241         }
 242 }
 243 EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
 244
 245 static void drop_user_return_notifiers(void *ignore)
 246 {
 247         unsigned int cpu = smp_processor_id();
 248         struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);
 249
 250         if (smsr->registered)
 251                 kvm_on_user_return(&smsr->urn);
 252 }
 253
 254 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 255 {
 256         return vcpu->arch.apic_base;
 257 }
 258 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 259
 260 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 261 {
 262         u64 old_state = vcpu->arch.apic_base &
 263                 (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
 264         u64 new_state = msr_info->data &
 265                 (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
 266         u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) |
 267                 0x2ff | (guest_cpuid_has_x2apic(vcpu) ? 0 : X2APIC_ENABLE);
 268
 269         if (!msr_info->host_initiated &&
 270             ((msr_info->data & reserved_bits) != 0 ||
 271              new_state == X2APIC_ENABLE ||
 272              (new_state == MSR_IA32_APICBASE_ENABLE &&
 273               old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) ||
 274              (new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) &&
 275               old_state == 0)))
 276                 return 1;
 277
 278         kvm_lapic_set_base(vcpu, msr_info->data);
 279         return 0;
 280 }
 281 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 282
 283 asmlinkage void kvm_spurious_fault(void)
 284 {
 285         /* Fault while not rebooting.  We want the trace. */
 286         BUG();
 287 }
 288 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
 289
 290 #define EXCPT_BENIGN            0
 291 #define EXCPT_CONTRIBUTORY      1
 292 #define EXCPT_PF                2
 293
 294 static int exception_class(int vector)
 295 {
 296         switch (vector) {
 297         case PF_VECTOR:
 298                 return EXCPT_PF;
 299         case DE_VECTOR:
 300         case TS_VECTOR:
 301         case NP_VECTOR:
 302         case SS_VECTOR:
 303         case GP_VECTOR:
 304                 return EXCPT_CONTRIBUTORY;
 305         default:
 306                 break;
 307         }
 308         return EXCPT_BENIGN;
 309 }
 310
 311 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
 312                 unsigned nr, bool has_error, u32 error_code,
 313                 bool reinject)
 314 {
 315         u32 prev_nr;
 316         int class1, class2;
 317
 318         kvm_make_request(KVM_REQ_EVENT, vcpu);
 319
 320         if (!vcpu->arch.exception.pending) {
 321         queue:
 322                 vcpu->arch.exception.pending = true;
 323                 vcpu->arch.exception.has_error_code = has_error;
 324                 vcpu->arch.exception.nr = nr;
 325                 vcpu->arch.exception.error_code = error_code;
 326                 vcpu->arch.exception.reinject = reinject;
 327                 return;
 328         }
 329
 330         /* to check exception */
 331         prev_nr = vcpu->arch.exception.nr;
 332         if (prev_nr == DF_VECTOR) {
 333                 /* triple fault -> shutdown */
 334                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
 335                 return;
 336         }
 337         class1 = exception_class(prev_nr);
 338         class2 = exception_class(nr);
 339         if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
 340                 || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
 341                 /* generate double fault per SDM Table 5-5 */
 342                 vcpu->arch.exception.pending = true;
 343                 vcpu->arch.exception.has_error_code = true;
 344                 vcpu->arch.exception.nr = DF_VECTOR;
 345                 vcpu->arch.exception.error_code = 0;
 346         } else
 347                 /* replace previous exception with a new one in a hope
 348                    that instruction re-execution will regenerate lost
 349                    exception */
 350                 goto queue;
 351 }
 352
 353 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 354 {
 355         kvm_multiple_exception(vcpu, nr, false, 0, false);
 356 }
 357 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 358
 359 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 360 {
 361         kvm_multiple_exception(vcpu, nr, false, 0, true);
 362 }
 363 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 364
 365 void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 366 {
 367         if (err)
 368                 kvm_inject_gp(vcpu, 0);
 369         else
 370                 kvm_x86_ops->skip_emulated_instruction(vcpu);
 371 }
 372 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
 373
 374 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 375 {
 376         ++vcpu->stat.pf_guest;
 377         vcpu->arch.cr2 = fault->address;
 378         kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
 379 }
 380 EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
 381
 382 void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
 383 {
 384         if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
 385                 vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
 386         else
 387                 vcpu->arch.mmu.inject_page_fault(vcpu, fault);
 388 }
 389
 390 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 391 {
 392         atomic_inc(&vcpu->arch.nmi_queued);
 393         kvm_make_request(KVM_REQ_NMI, vcpu);
 394 }
 395 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
 396
 397 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 398 {
 399         kvm_multiple_exception(vcpu, nr, true, error_code, false);
 400 }
 401 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 402
 403 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 404 {
 405         kvm_multiple_exception(vcpu, nr, true, error_code, true);
 406 }
 407 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
 408
 409 /*
 410  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 411  * a #GP and return false.
 412  */
 413 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
 414 {
 415         if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
 416                 return true;
 417         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
 418         return false;
 419 }
 420 EXPORT_SYMBOL_GPL(kvm_require_cpl);
 421
 422 /*
 423  * This function will be used to read from the physical memory of the currently
 424  * running guest. The difference to kvm_read_guest_page is that this function
 425  * can read from guest physical or from the guest's guest physical memory.
 426  */
 427 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 428                             gfn_t ngfn, void *data, int offset, int len,
 429                             u32 access)
 430 {
 431         gfn_t real_gfn;
 432         gpa_t ngpa;
 433
 434         ngpa     = gfn_to_gpa(ngfn);
 435         real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
 436         if (real_gfn == UNMAPPED_GVA)
 437                 return -EFAULT;
 438
 439         real_gfn = gpa_to_gfn(real_gfn);
 440
 441         return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
 442 }
 443 EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);
 444
 445 int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
 446                                void *data, int offset, int len, u32 access)
 447 {
 448         return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
 449                                        data, offset, len, access);
 450 }
 451
 452 /*
 453  * Load the pae pdptrs.  Return true is they are all valid.
 454  */
 455 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
 456 {
 457         gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 458         unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 459         int i;
 460         int ret;
 461         u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
 462
 463         ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
 464                                       offset * sizeof(u64), sizeof(pdpte),
 465                                       PFERR_USER_MASK|PFERR_WRITE_MASK);
 466         if (ret < 0) {
 467                 ret = 0;
 468                 goto out;
 469         }
 470         for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 471                 if (is_present_gpte(pdpte[i]) &&
 472                     (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
 473                         ret = 0;
 474                         goto out;
 475                 }
 476         }
 477         ret = 1;
 478
 479         memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
 480         __set_bit(VCPU_EXREG_PDPTR,
 481                   (unsigned long *)&vcpu->arch.regs_avail);
 482         __set_bit(VCPU_EXREG_PDPTR,
 483                   (unsigned long *)&vcpu->arch.regs_dirty);
 484 out:
 485
 486         return ret;
 487 }
 488 EXPORT_SYMBOL_GPL(load_pdptrs);
 489
 490 static bool pdptrs_changed(struct kvm_vcpu *vcpu)
 491 {
 492         u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
 493         bool changed = true;
 494         int offset;
 495         gfn_t gfn;
 496         int r;
 497
 498         if (is_long_mode(vcpu) || !is_pae(vcpu))
 499                 return false;
 500
 501         if (!test_bit(VCPU_EXREG_PDPTR,
 502                       (unsigned long *)&vcpu->arch.regs_avail))
 503                 return true;
 504
 505         gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
 506         offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
 507         r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
 508                                        PFERR_USER_MASK | PFERR_WRITE_MASK);
 509         if (r < 0)
 510                 goto out;
 511         changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
 512 out:
 513
 514         return changed;
 515 }
 516
 517 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 518 {
 519         unsigned long old_cr0 = kvm_read_cr0(vcpu);
 520         unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
 521                                     X86_CR0_CD | X86_CR0_NW;
 522
 523         cr0 |= X86_CR0_ET;
 524
 525 #ifdef CONFIG_X86_64
 526         if (cr0 & 0xffffffff00000000UL)
 527                 return 1;
 528 #endif
 529
 530         cr0 &= ~CR0_RESERVED_BITS;
 531
 532         if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
 533                 return 1;
 534
 535         if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
 536                 return 1;
 537
 538         if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 539 #ifdef CONFIG_X86_64
 540                 if ((vcpu->arch.efer & EFER_LME)) {
 541                         int cs_db, cs_l;
 542
 543                         if (!is_pae(vcpu))
 544                                 return 1;
 545                         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 546                         if (cs_l)
 547                                 return 1;
 548                 } else
 549 #endif
 550                 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
 551                                                  kvm_read_cr3(vcpu)))
 552                         return 1;
 553         }
 554
 555         if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
 556                 return 1;
 557
 558         kvm_x86_ops->set_cr0(vcpu, cr0);
 559
 560         if ((cr0 ^ old_cr0) & X86_CR0_PG) {
 561                 kvm_clear_async_pf_completion_queue(vcpu);
 562                 kvm_async_pf_hash_reset(vcpu);
 563         }
 564
 565         if ((cr0 ^ old_cr0) & update_bits)
 566                 kvm_mmu_reset_context(vcpu);
 567         return 0;
 568 }
 569 EXPORT_SYMBOL_GPL(kvm_set_cr0);
 570
 571 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 572 {
 573         (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
 574 }
 575 EXPORT_SYMBOL_GPL(kvm_lmsw);
 576
 577 static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
 578 {
 579         if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
 580                         !vcpu->guest_xcr0_loaded) {
 581                 /* kvm_set_xcr() also depends on this */
 582                 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
 583                 vcpu->guest_xcr0_loaded = 1;
 584         }
 585 }
 586
 587 static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
 588 {
 589         if (vcpu->guest_xcr0_loaded) {
 590                 if (vcpu->arch.xcr0 != host_xcr0)
 591                         xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
 592                 vcpu->guest_xcr0_loaded = 0;
 593         }
 594 }
 595
 596 int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
 597 {
 598         u64 xcr0 = xcr;
 599         u64 old_xcr0 = vcpu->arch.xcr0;
 600         u64 valid_bits;
 601
 602         /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
 603         if (index != XCR_XFEATURE_ENABLED_MASK)
 604                 return 1;
 605         if (!(xcr0 & XSTATE_FP))
 606                 return 1;
 607         if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
 608                 return 1;
 609
 610         /*
 611          * Do not allow the guest to set bits that we do not support
 612          * saving.  However, xcr0 bit 0 is always set, even if the
 613          * emulated CPU does not support XSAVE (see fx_init).
 614          */
 615         valid_bits = vcpu->arch.guest_supported_xcr0 | XSTATE_FP;
 616         if (xcr0 & ~valid_bits)
 617                 return 1;
 618
 619         if ((!(xcr0 & XSTATE_BNDREGS)) != (!(xcr0 & XSTATE_BNDCSR)))
 620                 return 1;
 621
 622         kvm_put_guest_xcr0(vcpu);
 623         vcpu->arch.xcr0 = xcr0;
 624
 625         if ((xcr0 ^ old_xcr0) & XSTATE_EXTEND_MASK)
 626                 kvm_update_cpuid(vcpu);
 627         return 0;
 628 }
 629
 630 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
 631 {
 632         if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
 633             __kvm_set_xcr(vcpu, index, xcr)) {
 634                 kvm_inject_gp(vcpu, 0);
 635                 return 1;
 636         }
 637         return 0;
 638 }
 639 EXPORT_SYMBOL_GPL(kvm_set_xcr);
 640
 641 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 642 {
 643         unsigned long old_cr4 = kvm_read_cr4(vcpu);
 644         unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
 645                                    X86_CR4_PAE | X86_CR4_SMEP;
 646         if (cr4 & CR4_RESERVED_BITS)
 647                 return 1;
 648
 649         if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
 650                 return 1;
 651
 652         if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
 653                 return 1;
 654
 655         if (!guest_cpuid_has_smap(vcpu) && (cr4 & X86_CR4_SMAP))
 656                 return 1;
 657
 658         if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_FSGSBASE))
 659                 return 1;
 660
 661         if (is_long_mode(vcpu)) {
 662                 if (!(cr4 & X86_CR4_PAE))
 663                         return 1;
 664         } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
 665                    && ((cr4 ^ old_cr4) & pdptr_bits)
 666                    && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
 667                                    kvm_read_cr3(vcpu)))
 668                 return 1;
 669
 670         if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
 671                 if (!guest_cpuid_has_pcid(vcpu))
 672                         return 1;
 673
 674                 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
 675                 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
 676                         return 1;
 677         }
 678
 679         if (kvm_x86_ops->set_cr4(vcpu, cr4))
 680                 return 1;
 681
 682         if (((cr4 ^ old_cr4) & pdptr_bits) ||
 683             (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
 684                 kvm_mmu_reset_context(vcpu);
 685
 686         if ((cr4 ^ old_cr4) & X86_CR4_SMAP)
 687                 update_permission_bitmask(vcpu, vcpu->arch.walk_mmu, false);
 688
 689         if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
 690                 kvm_update_cpuid(vcpu);
 691
 692         return 0;
 693 }
 694 EXPORT_SYMBOL_GPL(kvm_set_cr4);
 695
 696 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 697 {
 698         if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
 699                 kvm_mmu_sync_roots(vcpu);
 700                 kvm_mmu_flush_tlb(vcpu);
 701                 return 0;
 702         }
 703
 704         if (is_long_mode(vcpu) && (cr3 & CR3_L_MODE_RESERVED_BITS))
 705                 return 1;
 706         if (is_pae(vcpu) && is_paging(vcpu) &&
 707             !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
 708                 return 1;
 709
 710         vcpu->arch.cr3 = cr3;
 711         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
 712         kvm_mmu_new_cr3(vcpu);
 713         return 0;
 714 }
 715 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 716
 717 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 718 {
 719         if (cr8 & CR8_RESERVED_BITS)
 720                 return 1;
 721         if (irqchip_in_kernel(vcpu->kvm))
 722                 kvm_lapic_set_tpr(vcpu, cr8);
 723         else
 724                 vcpu->arch.cr8 = cr8;
 725         return 0;
 726 }
 727 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 728
 729 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 730 {
 731         if (irqchip_in_kernel(vcpu->kvm))
 732                 return kvm_lapic_get_cr8(vcpu);
 733         else
 734                 return vcpu->arch.cr8;
 735 }
 736 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 737
 738 static void kvm_update_dr6(struct kvm_vcpu *vcpu)
 739 {
 740         if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
 741                 kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
 742 }
 743
 744 static void kvm_update_dr7(struct kvm_vcpu *vcpu)
 745 {
 746         unsigned long dr7;
 747
 748         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
 749                 dr7 = vcpu->arch.guest_debug_dr7;
 750         else
 751                 dr7 = vcpu->arch.dr7;
 752         kvm_x86_ops->set_dr7(vcpu, dr7);
 753         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
 754         if (dr7 & DR7_BP_EN_MASK)
 755                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
 756 }
 757
 758 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 759 {
 760         switch (dr) {
 761         case 0 ... 3:
 762                 vcpu->arch.db[dr] = val;
 763                 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
 764                         vcpu->arch.eff_db[dr] = val;
 765                 break;
 766         case 4:
 767                 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 768                         return 1; /* #UD */
 769                 /* fall through */
 770         case 6:
 771                 if (val & 0xffffffff00000000ULL)
 772                         return -1; /* #GP */
 773                 vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
 774                 kvm_update_dr6(vcpu);
 775                 break;
 776         case 5:
 777                 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 778                         return 1; /* #UD */
 779                 /* fall through */
 780         default: /* 7 */
 781                 if (val & 0xffffffff00000000ULL)
 782                         return -1; /* #GP */
 783                 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
 784                 kvm_update_dr7(vcpu);
 785                 break;
 786         }
 787
 788         return 0;
 789 }
 790
 791 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 792 {
 793         int res;
 794
 795         res = __kvm_set_dr(vcpu, dr, val);
 796         if (res > 0)
 797                 kvm_queue_exception(vcpu, UD_VECTOR);
 798         else if (res < 0)
 799                 kvm_inject_gp(vcpu, 0);
 800
 801         return res;
 802 }
 803 EXPORT_SYMBOL_GPL(kvm_set_dr);
 804
 805 static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
 806 {
 807         switch (dr) {
 808         case 0 ... 3:
 809                 *val = vcpu->arch.db[dr];
 810                 break;
 811         case 4:
 812                 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 813                         return 1;
 814                 /* fall through */
 815         case 6:
 816                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
 817                         *val = vcpu->arch.dr6;
 818                 else
 819                         *val = kvm_x86_ops->get_dr6(vcpu);
 820                 break;
 821         case 5:
 822                 if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 823                         return 1;
 824                 /* fall through */
 825         default: /* 7 */
 826                 *val = vcpu->arch.dr7;
 827                 break;
 828         }
 829
 830         return 0;
 831 }
 832
 833 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
 834 {
 835         if (_kvm_get_dr(vcpu, dr, val)) {
 836                 kvm_queue_exception(vcpu, UD_VECTOR);
 837                 return 1;
 838         }
 839         return 0;
 840 }
 841 EXPORT_SYMBOL_GPL(kvm_get_dr);
 842
 843 bool kvm_rdpmc(struct kvm_vcpu *vcpu)
 844 {
 845         u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
 846         u64 data;
 847         int err;
 848
 849         err = kvm_pmu_read_pmc(vcpu, ecx, &data);
 850         if (err)
 851                 return err;
 852         kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
 853         kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
 854         return err;
 855 }
 856 EXPORT_SYMBOL_GPL(kvm_rdpmc);
 857
 858 /*
 859  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 860  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 861  *
 862  * This list is modified at module load time to reflect the
 863  * capabilities of the host cpu. This capabilities test skips MSRs that are
 864  * kvm-specific. Those are put in the beginning of the list.
 865  */
 866
 867 #define KVM_SAVE_MSRS_BEGIN     12
 868 static u32 msrs_to_save[] = {
 869         MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
 870         MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
 871         HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
 872         HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
 873         HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
 874         MSR_KVM_PV_EOI_EN,
 875         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 876         MSR_STAR,
 877 #ifdef CONFIG_X86_64
 878         MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 879 #endif
 880         MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
 881         MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS
 882 };
 883
 884 static unsigned num_msrs_to_save;
 885
 886 static const u32 emulated_msrs[] = {
 887         MSR_IA32_TSC_ADJUST,
 888         MSR_IA32_TSCDEADLINE,
 889         MSR_IA32_MISC_ENABLE,
 890         MSR_IA32_MCG_STATUS,
 891         MSR_IA32_MCG_CTL,
 892 };
 893
 894 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
 895 {
 896         if (efer & efer_reserved_bits)
 897                 return false;
 898
 899         if (efer & EFER_FFXSR) {
 900                 struct kvm_cpuid_entry2 *feat;
 901
 902                 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 903                 if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
 904                         return false;
 905         }
 906
 907         if (efer & EFER_SVME) {
 908                 struct kvm_cpuid_entry2 *feat;
 909
 910                 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 911                 if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
 912                         return false;
 913         }
 914
 915         return true;
 916 }
 917 EXPORT_SYMBOL_GPL(kvm_valid_efer);
 918
 919 static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
 920 {
 921         u64 old_efer = vcpu->arch.efer;
 922
 923         if (!kvm_valid_efer(vcpu, efer))
 924                 return 1;
 925
 926         if (is_paging(vcpu)
 927             && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
 928                 return 1;
 929
 930         efer &= ~EFER_LMA;
 931         efer |= vcpu->arch.efer & EFER_LMA;
 932
 933         kvm_x86_ops->set_efer(vcpu, efer);
 934
 935         /* Update reserved bits */
 936         if ((efer ^ old_efer) & EFER_NX)
 937                 kvm_mmu_reset_context(vcpu);
 938
 939         return 0;
 940 }
 941
 942 void kvm_enable_efer_bits(u64 mask)
 943 {
 944        efer_reserved_bits &= ~mask;
 945 }
 946 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
 947
 948
 949 /*
 950  * Writes msr value into into the appropriate "register".
 951  * Returns 0 on success, non-0 otherwise.
 952  * Assumes vcpu_load() was already called.
 953  */
 954 int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
 955 {
 956         return kvm_x86_ops->set_msr(vcpu, msr);
 957 }
 958
 959 /*
 960  * Adapt set_msr() to msr_io()'s calling convention
 961  */
 962 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
 963 {
 964         struct msr_data msr;
 965
 966         msr.data = *data;
 967         msr.index = index;
 968         msr.host_initiated = true;
 969         return kvm_set_msr(vcpu, &msr);
 970 }
 971
 972 #ifdef CONFIG_X86_64
 973 struct pvclock_gtod_data {
 974         seqcount_t      seq;
 975
 976         struct { /* extract of a clocksource struct */
 977                 int vclock_mode;
 978                 cycle_t cycle_last;
 979                 cycle_t mask;
 980                 u32     mult;
 981                 u32     shift;
 982         } clock;
 983
 984         /* open coded 'struct timespec' */
 985         u64             monotonic_time_snsec;
 986         time_t          monotonic_time_sec;
 987 };
 988
 989 static struct pvclock_gtod_data pvclock_gtod_data;
 990
 991 static void update_pvclock_gtod(struct timekeeper *tk)
 992 {
 993         struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
 994
 995         write_seqcount_begin(&vdata->seq);
 996
 997         /* copy pvclock gtod data */
 998         vdata->clock.vclock_mode        = tk->clock->archdata.vclock_mode;
 999         vdata->clock.cycle_last         = tk->clock->cycle_last;
1000         vdata->clock.mask               = tk->clock->mask;
1001         vdata->clock.mult               = tk->mult;
1002         vdata->clock.shift              = tk->shift;
1003
1004         vdata->monotonic_time_sec       = tk->xtime_sec
1005                                         + tk->wall_to_monotonic.tv_sec;
1006         vdata->monotonic_time_snsec     = tk->xtime_nsec
1007                                         + (tk->wall_to_monotonic.tv_nsec
1008                                                 << tk->shift);
1009         while (vdata->monotonic_time_snsec >=
1010                                         (((u64)NSEC_PER_SEC) << tk->shift)) {
1011                 vdata->monotonic_time_snsec -=
1012                                         ((u64)NSEC_PER_SEC) << tk->shift;
1013                 vdata->monotonic_time_sec++;
1014         }
1015
1016         write_seqcount_end(&vdata->seq);
1017 }
1018 #endif
1019
1020
1021 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
1022 {
1023         int version;
1024         int r;
1025         struct pvclock_wall_clock wc;
1026         struct timespec boot;
1027
1028         if (!wall_clock)
1029                 return;
1030
1031         r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
1032         if (r)
1033                 return;
1034
1035         if (version & 1)
1036                 ++version;  /* first time write, random junk */
1037
1038         ++version;
1039
1040         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
1041
1042         /*
1043          * The guest calculates current wall clock time by adding
1044          * system time (updated by kvm_guest_time_update below) to the
1045          * wall clock specified here.  guest system time equals host
1046          * system time for us, thus we must fill in host boot time here.
1047          */
1048         getboottime(&boot);
1049
1050         if (kvm->arch.kvmclock_offset) {
1051                 struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset);
1052                 boot = timespec_sub(boot, ts);
1053         }
1054         wc.sec = boot.tv_sec;
1055         wc.nsec = boot.tv_nsec;
1056         wc.version = version;
1057
1058         kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
1059
1060         version++;
1061         kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
1062 }
1063
1064 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
1065 {
1066         uint32_t quotient, remainder;
1067
1068         /* Don't try to replace with do_div(), this one calculates
1069          * "(dividend << 32) / divisor" */
1070         __asm__ ( "divl %4"
1071                   : "=a" (quotient), "=d" (remainder)
1072                   : "0" (0), "1" (dividend), "r" (divisor) );
1073         return quotient;
1074 }
1075
1076 static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
1077                                s8 *pshift, u32 *pmultiplier)
1078 {
1079         uint64_t scaled64;
1080         int32_t  shift = 0;
1081         uint64_t tps64;
1082         uint32_t tps32;
1083
1084         tps64 = base_khz * 1000LL;
1085         scaled64 = scaled_khz * 1000LL;
1086         while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
1087                 tps64 >>= 1;
1088                 shift--;
1089         }
1090
1091         tps32 = (uint32_t)tps64;
1092         while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
1093                 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
1094                         scaled64 >>= 1;
1095                 else
1096                         tps32 <<= 1;
1097                 shift++;
1098         }
1099
1100         *pshift = shift;
1101         *pmultiplier = div_frac(scaled64, tps32);
1102
1103         pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
1104                  __func__, base_khz, scaled_khz, shift, *pmultiplier);
1105 }
1106
1107 static inline u64 get_kernel_ns(void)
1108 {
1109         struct timespec ts;
1110
1111         ktime_get_ts(&ts);
1112         monotonic_to_bootbased(&ts);
1113         return timespec_to_ns(&ts);
1114 }
1115
1116 #ifdef CONFIG_X86_64
1117 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
1118 #endif
1119
1120 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
1121 unsigned long max_tsc_khz;
1122
1123 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
1124 {
1125         return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
1126                                    vcpu->arch.virtual_tsc_shift);
1127 }
1128
1129 static u32 adjust_tsc_khz(u32 khz, s32 ppm)
1130 {
1131         u64 v = (u64)khz * (1000000 + ppm);
1132         do_div(v, 1000000);
1133         return v;
1134 }
1135
1136 static void kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
1137 {
1138         u32 thresh_lo, thresh_hi;
1139         int use_scaling = 0;
1140
1141         /* tsc_khz can be zero if TSC calibration fails */
1142         if (this_tsc_khz == 0)
1143                 return;
1144
1145         /* Compute a scale to convert nanoseconds in TSC cycles */
1146         kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
1147                            &vcpu->arch.virtual_tsc_shift,
1148                            &vcpu->arch.virtual_tsc_mult);
1149         vcpu->arch.virtual_tsc_khz = this_tsc_khz;
1150
1151         /*
1152          * Compute the variation in TSC rate which is acceptable
1153          * within the range of tolerance and decide if the
1154          * rate being applied is within that bounds of the hardware
1155          * rate.  If so, no scaling or compensation need be done.
1156          */
1157         thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
1158         thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
1159         if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) {
1160                 pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi);
1161                 use_scaling = 1;
1162         }
1163         kvm_x86_ops->set_tsc_khz(vcpu, this_tsc_khz, use_scaling);
1164 }
1165
1166 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
1167 {
1168         u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
1169                                       vcpu->arch.virtual_tsc_mult,
1170                                       vcpu->arch.virtual_tsc_shift);
1171         tsc += vcpu->arch.this_tsc_write;
1172         return tsc;
1173 }
1174
1175 void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
1176 {
1177 #ifdef CONFIG_X86_64
1178         bool vcpus_matched;
1179         bool do_request = false;
1180         struct kvm_arch *ka = &vcpu->kvm->arch;
1181         struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1182
1183         vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
1184                          atomic_read(&vcpu->kvm->online_vcpus));
1185
1186         if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC)
1187                 if (!ka->use_master_clock)
1188                         do_request = 1;
1189
1190         if (!vcpus_matched && ka->use_master_clock)
1191                         do_request = 1;
1192
1193         if (do_request)
1194                 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
1195
1196         trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
1197                             atomic_read(&vcpu->kvm->online_vcpus),
1198                             ka->use_master_clock, gtod->clock.vclock_mode);
1199 #endif
1200 }
1201
1202 static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
1203 {
1204         u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
1205         vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
1206 }
1207
1208 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
1209 {
1210         struct kvm *kvm = vcpu->kvm;
1211         u64 offset, ns, elapsed;
1212         unsigned long flags;
1213         s64 usdiff;
1214         bool matched;
1215         u64 data = msr->data;
1216
1217         raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
1218         offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1219         ns = get_kernel_ns();
1220         elapsed = ns - kvm->arch.last_tsc_nsec;
1221
1222         if (vcpu->arch.virtual_tsc_khz) {
1223                 int faulted = 0;
1224
1225                 /* n.b - signed multiplication and division required */
1226                 usdiff = data - kvm->arch.last_tsc_write;
1227 #ifdef CONFIG_X86_64
1228                 usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz;
1229 #else
1230                 /* do_div() only does unsigned */
1231                 asm("1: idivl %[divisor]\n"
1232                     "2: xor %%edx, %%edx\n"
1233                     "   movl $0, %[faulted]\n"
1234                     "3:\n"
1235                     ".section .fixup,\"ax\"\n"
1236                     "4: movl $1, %[faulted]\n"
1237                     "   jmp  3b\n"
1238                     ".previous\n"
1239
1240                 _ASM_EXTABLE(1b, 4b)
1241
1242                 : "=A"(usdiff), [faulted] "=r" (faulted)
1243                 : "A"(usdiff * 1000), [divisor] "rm"(vcpu->arch.virtual_tsc_khz));
1244
1245 #endif
1246                 do_div(elapsed, 1000);
1247                 usdiff -= elapsed;
1248                 if (usdiff < 0)
1249                         usdiff = -usdiff;
1250
1251                 /* idivl overflow => difference is larger than USEC_PER_SEC */
1252                 if (faulted)
1253                         usdiff = USEC_PER_SEC;
1254         } else
1255                 usdiff = USEC_PER_SEC; /* disable TSC match window below */
1256
1257         /*
1258          * Special case: TSC write with a small delta (1 second) of virtual
1259          * cycle time against real time is interpreted as an attempt to
1260          * synchronize the CPU.
1261          *
1262          * For a reliable TSC, we can match TSC offsets, and for an unstable
1263          * TSC, we add elapsed time in this computation.  We could let the
1264          * compensation code attempt to catch up if we fall behind, but
1265          * it's better to try to match offsets from the beginning.
1266          */
1267         if (usdiff < USEC_PER_SEC &&
1268             vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
1269                 if (!check_tsc_unstable()) {
1270                         offset = kvm->arch.cur_tsc_offset;
1271                         pr_debug("kvm: matched tsc offset for %llu\n", data);
1272                 } else {
1273                         u64 delta = nsec_to_cycles(vcpu, elapsed);
1274                         data += delta;
1275                         offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
1276                         pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
1277                 }
1278                 matched = true;
1279         } else {
1280                 /*
1281                  * We split periods of matched TSC writes into generations.
1282                  * For each generation, we track the original measured
1283                  * nanosecond time, offset, and write, so if TSCs are in
1284                  * sync, we can match exact offset, and if not, we can match
1285                  * exact software computation in compute_guest_tsc()
1286                  *
1287                  * These values are tracked in kvm->arch.cur_xxx variables.
1288                  */
1289                 kvm->arch.cur_tsc_generation++;
1290                 kvm->arch.cur_tsc_nsec = ns;
1291                 kvm->arch.cur_tsc_write = data;
1292                 kvm->arch.cur_tsc_offset = offset;
1293                 matched = false;
1294                 pr_debug("kvm: new tsc generation %u, clock %llu\n",
1295                          kvm->arch.cur_tsc_generation, data);
1296         }
1297
1298         /*
1299          * We also track th most recent recorded KHZ, write and time to
1300          * allow the matching interval to be extended at each write.
1301          */
1302         kvm->arch.last_tsc_nsec = ns;
1303         kvm->arch.last_tsc_write = data;
1304         kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
1305
1306         vcpu->arch.last_guest_tsc = data;
1307
1308         /* Keep track of which generation this VCPU has synchronized to */
1309         vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
1310         vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
1311         vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
1312
1313         if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
1314                 update_ia32_tsc_adjust_msr(vcpu, offset);
1315         kvm_x86_ops->write_tsc_offset(vcpu, offset);
1316         raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
1317
1318         spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
1319         if (matched)
1320                 kvm->arch.nr_vcpus_matched_tsc++;
1321         else
1322                 kvm->arch.nr_vcpus_matched_tsc = 0;
1323
1324         kvm_track_tsc_matching(vcpu);
1325         spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
1326 }
1327
1328 EXPORT_SYMBOL_GPL(kvm_write_tsc);
1329
1330 #ifdef CONFIG_X86_64
1331
1332 static cycle_t read_tsc(void)
1333 {
1334         cycle_t ret;
1335         u64 last;
1336
1337         /*
1338          * Empirically, a fence (of type that depends on the CPU)
1339          * before rdtsc is enough to ensure that rdtsc is ordered
1340          * with respect to loads.  The various CPU manuals are unclear
1341          * as to whether rdtsc can be reordered with later loads,
1342          * but no one has ever seen it happen.
1343          */
1344         rdtsc_barrier();
1345         ret = (cycle_t)vget_cycles();
1346
1347         last = pvclock_gtod_data.clock.cycle_last;
1348
1349         if (likely(ret >= last))
1350                 return ret;
1351
1352         /*
1353          * GCC likes to generate cmov here, but this branch is extremely
1354          * predictable (it's just a funciton of time and the likely is
1355          * very likely) and there's a data dependence, so force GCC
1356          * to generate a branch instead.  I don't barrier() because
1357          * we don't actually need a barrier, and if this function
1358          * ever gets inlined it will generate worse code.
1359          */
1360         asm volatile ("");
1361         return last;
1362 }
1363
1364 static inline u64 vgettsc(cycle_t *cycle_now)
1365 {
1366         long v;
1367         struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1368
1369         *cycle_now = read_tsc();
1370
1371         v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
1372         return v * gtod->clock.mult;
1373 }
1374
1375 static int do_monotonic(struct timespec *ts, cycle_t *cycle_now)
1376 {
1377         unsigned long seq;
1378         u64 ns;
1379         int mode;
1380         struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
1381
1382         ts->tv_nsec = 0;
1383         do {
1384                 seq = read_seqcount_begin(&gtod->seq);
1385                 mode = gtod->clock.vclock_mode;
1386                 ts->tv_sec = gtod->monotonic_time_sec;
1387                 ns = gtod->monotonic_time_snsec;
1388                 ns += vgettsc(cycle_now);
1389                 ns >>= gtod->clock.shift;
1390         } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
1391         timespec_add_ns(ts, ns);
1392
1393         return mode;
1394 }
1395
1396 /* returns true if host is using tsc clocksource */
1397 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
1398 {
1399         struct timespec ts;
1400
1401         /* checked again under seqlock below */
1402         if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
1403                 return false;
1404
1405         if (do_monotonic(&ts, cycle_now) != VCLOCK_TSC)
1406                 return false;
1407
1408         monotonic_to_bootbased(&ts);
1409         *kernel_ns = timespec_to_ns(&ts);
1410
1411         return true;
1412 }
1413 #endif
1414
1415 /*
1416  *
1417  * Assuming a stable TSC across physical CPUS, and a stable TSC
1418  * across virtual CPUs, the following condition is possible.
1419  * Each numbered line represents an event visible to both
1420  * CPUs at the next numbered event.
1421  *
1422  * "timespecX" represents host monotonic time. "tscX" represents
1423  * RDTSC value.
1424  *
1425  *              VCPU0 on CPU0           |       VCPU1 on CPU1
1426  *
1427  * 1.  read timespec0,tsc0
1428  * 2.                                   | timespec1 = timespec0 + N
1429  *                                      | tsc1 = tsc0 + M
1430  * 3. transition to guest               | transition to guest
1431  * 4. ret0 = timespec0 + (rdtsc - tsc0) |
1432  * 5.                                   | ret1 = timespec1 + (rdtsc - tsc1)
1433  *                                      | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
1434  *
1435  * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
1436  *
1437  *      - ret0 < ret1
1438  *      - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
1439  *              ...
1440  *      - 0 < N - M => M < N
1441  *
1442  * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
1443  * always the case (the difference between two distinct xtime instances
1444  * might be smaller then the difference between corresponding TSC reads,
1445  * when updating guest vcpus pvclock areas).
1446  *
1447  * To avoid that problem, do not allow visibility of distinct
1448  * system_timestamp/tsc_timestamp values simultaneously: use a master
1449  * copy of host monotonic time values. Update that master copy
1450  * in lockstep.
1451  *
1452  * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
1453  *
1454  */
1455
1456 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
1457 {
1458 #ifdef CONFIG_X86_64
1459         struct kvm_arch *ka = &kvm->arch;
1460         int vclock_mode;
1461         bool host_tsc_clocksource, vcpus_matched;
1462
1463         vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
1464                         atomic_read(&kvm->online_vcpus));
1465
1466         /*
1467          * If the host uses TSC clock, then passthrough TSC as stable
1468          * to the guest.
1469          */
1470         host_tsc_clocksource = kvm_get_time_and_clockread(
1471                                         &ka->master_kernel_ns,
1472                                         &ka->master_cycle_now);
1473
1474         ka->use_master_clock = host_tsc_clocksource & vcpus_matched;
1475
1476         if (ka->use_master_clock)
1477                 atomic_set(&kvm_guest_has_master_clock, 1);
1478
1479         vclock_mode = pvclock_gtod_data.clock.vclock_mode;
1480         trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
1481                                         vcpus_matched);
1482 #endif
1483 }
1484
1485 static void kvm_gen_update_masterclock(struct kvm *kvm)
1486 {
1487 #ifdef CONFIG_X86_64
1488         int i;
1489         struct kvm_vcpu *vcpu;
1490         struct kvm_arch *ka = &kvm->arch;
1491
1492         spin_lock(&ka->pvclock_gtod_sync_lock);
1493         kvm_make_mclock_inprogress_request(kvm);
1494         /* no guest entries from this point */
1495         pvclock_update_vm_gtod_copy(kvm);
1496
1497         kvm_for_each_vcpu(i, vcpu, kvm)
1498                 set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
1499
1500         /* guest entries allowed */
1501         kvm_for_each_vcpu(i, vcpu, kvm)
1502                 clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests);
1503
1504         spin_unlock(&ka->pvclock_gtod_sync_lock);
1505 #endif
1506 }
1507
1508 static int kvm_guest_time_update(struct kvm_vcpu *v)
1509 {
1510         unsigned long flags, this_tsc_khz;
1511         struct kvm_vcpu_arch *vcpu = &v->arch;
1512         struct kvm_arch *ka = &v->kvm->arch;
1513         s64 kernel_ns;
1514         u64 tsc_timestamp, host_tsc;
1515         struct pvclock_vcpu_time_info guest_hv_clock;
1516         u8 pvclock_flags;
1517         bool use_master_clock;
1518
1519         kernel_ns = 0;
1520         host_tsc = 0;
1521
1522         /*
1523          * If the host uses TSC clock, then passthrough TSC as stable
1524          * to the guest.
1525          */
1526         spin_lock(&ka->pvclock_gtod_sync_lock);
1527         use_master_clock = ka->use_master_clock;
1528         if (use_master_clock) {
1529                 host_tsc = ka->master_cycle_now;
1530                 kernel_ns = ka->master_kernel_ns;
1531         }
1532         spin_unlock(&ka->pvclock_gtod_sync_lock);
1533
1534         /* Keep irq disabled to prevent changes to the clock */
1535         local_irq_save(flags);
1536         this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
1537         if (unlikely(this_tsc_khz == 0)) {
1538                 local_irq_restore(flags);
1539                 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
1540                 return 1;
1541         }
1542         if (!use_master_clock) {
1543                 host_tsc = native_read_tsc();
1544                 kernel_ns = get_kernel_ns();
1545         }
1546
1547         tsc_timestamp = kvm_x86_ops->read_l1_tsc(v, host_tsc);
1548
1549         /*
1550          * We may have to catch up the TSC to match elapsed wall clock
1551          * time for two reasons, even if kvmclock is used.
1552          *   1) CPU could have been running below the maximum TSC rate
1553          *   2) Broken TSC compensation resets the base at each VCPU
1554          *      entry to avoid unknown leaps of TSC even when running
1555          *      again on the same CPU.  This may cause apparent elapsed
1556          *      time to disappear, and the guest to stand still or run
1557          *      very slowly.
1558          */
1559         if (vcpu->tsc_catchup) {
1560                 u64 tsc = compute_guest_tsc(v, kernel_ns);
1561                 if (tsc > tsc_timestamp) {
1562                         adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
1563                         tsc_timestamp = tsc;
1564                 }
1565         }
1566
1567         local_irq_restore(flags);
1568
1569         if (!vcpu->pv_time_enabled)
1570                 return 0;
1571
1572         if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
1573                 kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
1574                                    &vcpu->hv_clock.tsc_shift,
1575                                    &vcpu->hv_clock.tsc_to_system_mul);
1576                 vcpu->hw_tsc_khz = this_tsc_khz;
1577         }
1578
1579         /* With all the info we got, fill in the values */
1580         vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
1581         vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
1582         vcpu->last_guest_tsc = tsc_timestamp;
1583
1584         /*
1585          * The interface expects us to write an even number signaling that the
1586          * update is finished. Since the guest won't see the intermediate
1587          * state, we just increase by 2 at the end.
1588          */
1589         vcpu->hv_clock.version += 2;
1590
1591         if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
1592                 &guest_hv_clock, sizeof(guest_hv_clock))))
1593                 return 0;
1594
1595         /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
1596         pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);
1597
1598         if (vcpu->pvclock_set_guest_stopped_request) {
1599                 pvclock_flags |= PVCLOCK_GUEST_STOPPED;
1600                 vcpu->pvclock_set_guest_stopped_request = false;
1601         }
1602
1603         /* If the host uses TSC clocksource, then it is stable */
1604         if (use_master_clock)
1605                 pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
1606
1607         vcpu->hv_clock.flags = pvclock_flags;
1608
1609         kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
1610                                 &vcpu->hv_clock,
1611                                 sizeof(vcpu->hv_clock));
1612         return 0;
1613 }
1614
1615 /*
1616  * kvmclock updates which are isolated to a given vcpu, such as
1617  * vcpu->cpu migration, should not allow system_timestamp from
1618  * the rest of the vcpus to remain static. Otherwise ntp frequency
1619  * correction applies to one vcpu's system_timestamp but not
1620  * the others.
1621  *
1622  * So in those cases, request a kvmclock update for all vcpus.
1623  * We need to rate-limit these requests though, as they can
1624  * considerably slow guests that have a large number of vcpus.
1625  * The time for a remote vcpu to update its kvmclock is bound
1626  * by the delay we use to rate-limit the updates.
1627  */
1628
1629 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
1630
1631 static void kvmclock_update_fn(struct work_struct *work)
1632 {
1633         int i;
1634         struct delayed_work *dwork = to_delayed_work(work);
1635         struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
1636                                            kvmclock_update_work);
1637         struct kvm *kvm = container_of(ka, struct kvm, arch);
1638         struct kvm_vcpu *vcpu;
1639
1640         kvm_for_each_vcpu(i, vcpu, kvm) {
1641                 set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
1642                 kvm_vcpu_kick(vcpu);
1643         }
1644 }
1645
1646 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
1647 {
1648         struct kvm *kvm = v->kvm;
1649
1650         set_bit(KVM_REQ_CLOCK_UPDATE, &v->requests);
1651         schedule_delayed_work(&kvm->arch.kvmclock_update_work,
1652                                         KVMCLOCK_UPDATE_DELAY);
1653 }
1654
1655 #define KVMCLOCK_SYNC_PERIOD (300 * HZ)
1656
1657 static void kvmclock_sync_fn(struct work_struct *work)
1658 {
1659         struct delayed_work *dwork = to_delayed_work(work);
1660         struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
1661                                            kvmclock_sync_work);
1662         struct kvm *kvm = container_of(ka, struct kvm, arch);
1663
1664         schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
1665         schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
1666                                         KVMCLOCK_SYNC_PERIOD);
1667 }
1668
1669 static bool msr_mtrr_valid(unsigned msr)
1670 {
1671         switch (msr) {
1672         case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
1673         case MSR_MTRRfix64K_00000:
1674         case MSR_MTRRfix16K_80000:
1675         case MSR_MTRRfix16K_A0000:
1676         case MSR_MTRRfix4K_C0000:
1677         case MSR_MTRRfix4K_C8000:
1678         case MSR_MTRRfix4K_D0000:
1679         case MSR_MTRRfix4K_D8000:
1680         case MSR_MTRRfix4K_E0000:
1681         case MSR_MTRRfix4K_E8000:
1682         case MSR_MTRRfix4K_F0000:
1683         case MSR_MTRRfix4K_F8000:
1684         case MSR_MTRRdefType:
1685         case MSR_IA32_CR_PAT:
1686                 return true;
1687         case 0x2f8:
1688                 return true;
1689         }
1690         return false;
1691 }
1692
1693 static bool valid_pat_type(unsigned t)
1694 {
1695         return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
1696 }
1697
1698 static bool valid_mtrr_type(unsigned t)
1699 {
1700         return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
1701 }
1702
1703 static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1704 {
1705         int i;
1706
1707         if (!msr_mtrr_valid(msr))
1708                 return false;
1709
1710         if (msr == MSR_IA32_CR_PAT) {
1711                 for (i = 0; i < 8; i++)
1712                         if (!valid_pat_type((data >> (i * 8)) & 0xff))
1713                                 return false;
1714                 return true;
1715         } else if (msr == MSR_MTRRdefType) {
1716                 if (data & ~0xcff)
1717                         return false;
1718                 return valid_mtrr_type(data & 0xff);
1719         } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
1720                 for (i = 0; i < 8 ; i++)
1721                         if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
1722                                 return false;
1723                 return true;
1724         }
1725
1726         /* variable MTRRs */
1727         return valid_mtrr_type(data & 0xff);
1728 }
1729
1730 static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1731 {
1732         u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
1733
1734         if (!mtrr_valid(vcpu, msr, data))
1735                 return 1;
1736
1737         if (msr == MSR_MTRRdefType) {
1738                 vcpu->arch.mtrr_state.def_type = data;
1739                 vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
1740         } else if (msr == MSR_MTRRfix64K_00000)
1741                 p[0] = data;
1742         else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
1743                 p[1 + msr - MSR_MTRRfix16K_80000] = data;
1744         else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
1745                 p[3 + msr - MSR_MTRRfix4K_C0000] = data;
1746         else if (msr == MSR_IA32_CR_PAT)
1747                 vcpu->arch.pat = data;
1748         else {  /* Variable MTRRs */
1749                 int idx, is_mtrr_mask;
1750                 u64 *pt;
1751
1752                 idx = (msr - 0x200) / 2;
1753                 is_mtrr_mask = msr - 0x200 - 2 * idx;
1754                 if (!is_mtrr_mask)
1755                         pt =
1756                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
1757                 else
1758                         pt =
1759                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
1760                 *pt = data;
1761         }
1762
1763         kvm_mmu_reset_context(vcpu);
1764         return 0;
1765 }
1766
1767 static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1768 {
1769         u64 mcg_cap = vcpu->arch.mcg_cap;
1770         unsigned bank_num = mcg_cap & 0xff;
1771
1772         switch (msr) {
1773         case MSR_IA32_MCG_STATUS:
1774                 vcpu->arch.mcg_status = data;
1775                 break;
1776         case MSR_IA32_MCG_CTL:
1777                 if (!(mcg_cap & MCG_CTL_P))
1778                         return 1;
1779                 if (data != 0 && data != ~(u64)0)
1780                         return -1;
1781                 vcpu->arch.mcg_ctl = data;
1782                 break;
1783         default:
1784                 if (msr >= MSR_IA32_MC0_CTL &&
1785                     msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
1786                         u32 offset = msr - MSR_IA32_MC0_CTL;
1787                         /* only 0 or all 1s can be written to IA32_MCi_CTL
1788                          * some Linux kernels though clear bit 10 in bank 4 to
1789                          * workaround a BIOS/GART TBL issue on AMD K8s, ignore
1790                          * this to avoid an uncatched #GP in the guest
1791                          */
1792                         if ((offset & 0x3) == 0 &&
1793                             data != 0 && (data | (1 << 10)) != ~(u64)0)
1794                                 return -1;
1795                         vcpu->arch.mce_banks[offset] = data;
1796                         break;
1797                 }
1798                 return 1;
1799         }
1800         return 0;
1801 }
1802
1803 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
1804 {
1805         struct kvm *kvm = vcpu->kvm;
1806         int lm = is_long_mode(vcpu);
1807         u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
1808                 : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
1809         u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
1810                 : kvm->arch.xen_hvm_config.blob_size_32;
1811         u32 page_num = data & ~PAGE_MASK;
1812         u64 page_addr = data & PAGE_MASK;
1813         u8 *page;
1814         int r;
1815
1816         r = -E2BIG;
1817         if (page_num >= blob_size)
1818                 goto out;
1819         r = -ENOMEM;
1820         page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
1821         if (IS_ERR(page)) {
1822                 r = PTR_ERR(page);
1823                 goto out;
1824         }
1825         if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
1826                 goto out_free;
1827         r = 0;
1828 out_free:
1829         kfree(page);
1830 out:
1831         return r;
1832 }
1833
1834 static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
1835 {
1836         return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
1837 }
1838
1839 static bool kvm_hv_msr_partition_wide(u32 msr)
1840 {
1841         bool r = false;
1842         switch (msr) {
1843         case HV_X64_MSR_GUEST_OS_ID:
1844         case HV_X64_MSR_HYPERCALL:
1845         case HV_X64_MSR_REFERENCE_TSC:
1846         case HV_X64_MSR_TIME_REF_COUNT:
1847                 r = true;
1848                 break;
1849         }
1850
1851         return r;
1852 }
1853
1854 static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1855 {
1856         struct kvm *kvm = vcpu->kvm;
1857
1858         switch (msr) {
1859         case HV_X64_MSR_GUEST_OS_ID:
1860                 kvm->arch.hv_guest_os_id = data;
1861                 /* setting guest os id to zero disables hypercall page */
1862                 if (!kvm->arch.hv_guest_os_id)
1863                         kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
1864                 break;
1865         case HV_X64_MSR_HYPERCALL: {
1866                 u64 gfn;
1867                 unsigned long addr;
1868                 u8 instructions[4];
1869
1870                 /* if guest os id is not set hypercall should remain disabled */
1871                 if (!kvm->arch.hv_guest_os_id)
1872                         break;
1873                 if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
1874                         kvm->arch.hv_hypercall = data;
1875                         break;
1876                 }
1877                 gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
1878                 addr = gfn_to_hva(kvm, gfn);
1879                 if (kvm_is_error_hva(addr))
1880                         return 1;
1881                 kvm_x86_ops->patch_hypercall(vcpu, instructions);
1882                 ((unsigned char *)instructions)[3] = 0xc3; /* ret */
1883                 if (__copy_to_user((void __user *)addr, instructions, 4))
1884                         return 1;
1885                 kvm->arch.hv_hypercall = data;
1886                 mark_page_dirty(kvm, gfn);
1887                 break;
1888         }
1889         case HV_X64_MSR_REFERENCE_TSC: {
1890                 u64 gfn;
1891                 HV_REFERENCE_TSC_PAGE tsc_ref;
1892                 memset(&tsc_ref, 0, sizeof(tsc_ref));
1893                 kvm->arch.hv_tsc_page = data;
1894                 if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE))
1895                         break;
1896                 gfn = data >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
1897                 if (kvm_write_guest(kvm, data,
1898                         &tsc_ref, sizeof(tsc_ref)))
1899                         return 1;
1900                 mark_page_dirty(kvm, gfn);
1901                 break;
1902         }
1903         default:
1904                 vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
1905                             "data 0x%llx\n", msr, data);
1906                 return 1;
1907         }
1908         return 0;
1909 }
1910
1911 static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1912 {
1913         switch (msr) {
1914         case HV_X64_MSR_APIC_ASSIST_PAGE: {
1915                 u64 gfn;
1916                 unsigned long addr;
1917
1918                 if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
1919                         vcpu->arch.hv_vapic = data;
1920                         if (kvm_lapic_enable_pv_eoi(vcpu, 0))
1921                                 return 1;
1922                         break;
1923                 }
1924                 gfn = data >> HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT;
1925                 addr = gfn_to_hva(vcpu->kvm, gfn);
1926                 if (kvm_is_error_hva(addr))
1927                         return 1;
1928                 if (__clear_user((void __user *)addr, PAGE_SIZE))
1929                         return 1;
1930                 vcpu->arch.hv_vapic = data;
1931                 mark_page_dirty(vcpu->kvm, gfn);
1932                 if (kvm_lapic_enable_pv_eoi(vcpu, gfn_to_gpa(gfn) | KVM_MSR_ENABLED))
1933                         return 1;
1934                 break;
1935         }
1936         case HV_X64_MSR_EOI:
1937                 return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
1938         case HV_X64_MSR_ICR:
1939                 return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
1940         case HV_X64_MSR_TPR:
1941                 return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
1942         default:
1943                 vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
1944                             "data 0x%llx\n", msr, data);
1945                 return 1;
1946         }
1947
1948         return 0;
1949 }
1950
1951 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
1952 {
1953         gpa_t gpa = data & ~0x3f;
1954
1955         /* Bits 2:5 are reserved, Should be zero */
1956         if (data & 0x3c)
1957                 return 1;
1958
1959         vcpu->arch.apf.msr_val = data;
1960
1961         if (!(data & KVM_ASYNC_PF_ENABLED)) {
1962                 kvm_clear_async_pf_completion_queue(vcpu);
1963                 kvm_async_pf_hash_reset(vcpu);
1964                 return 0;
1965         }
1966
1967         if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
1968                                         sizeof(u32)))
1969                 return 1;
1970
1971         vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
1972         kvm_async_pf_wakeup_all(vcpu);
1973         return 0;
1974 }
1975
1976 static void kvmclock_reset(struct kvm_vcpu *vcpu)
1977 {
1978         vcpu->arch.pv_time_enabled = false;
1979 }
1980
1981 static void accumulate_steal_time(struct kvm_vcpu *vcpu)
1982 {
1983         u64 delta;
1984
1985         if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
1986                 return;
1987
1988         delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
1989         vcpu->arch.st.last_steal = current->sched_info.run_delay;
1990         vcpu->arch.st.accum_steal = delta;
1991 }
1992
1993 static void record_steal_time(struct kvm_vcpu *vcpu)
1994 {
1995         if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
1996                 return;
1997
1998         if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
1999                 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
2000                 return;
2001
2002         vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
2003         vcpu->arch.st.steal.version += 2;
2004         vcpu->arch.st.accum_steal = 0;
2005
2006         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
2007                 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
2008 }
2009
2010 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2011 {
2012         bool pr = false;
2013         u32 msr = msr_info->index;
2014         u64 data = msr_info->data;
2015
2016         switch (msr) {
2017         case MSR_AMD64_NB_CFG:
2018         case MSR_IA32_UCODE_REV:
2019         case MSR_IA32_UCODE_WRITE:
2020         case MSR_VM_HSAVE_PA:
2021         case MSR_AMD64_PATCH_LOADER:
2022         case MSR_AMD64_BU_CFG2:
2023                 break;
2024
2025         case MSR_EFER:
2026                 return set_efer(vcpu, data);
2027         case MSR_K7_HWCR:
2028                 data &= ~(u64)0x40;     /* ignore flush filter disable */
2029                 data &= ~(u64)0x100;    /* ignore ignne emulation enable */
2030                 data &= ~(u64)0x8;      /* ignore TLB cache disable */
2031                 if (data != 0) {
2032                         vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
2033                                     data);
2034                         return 1;
2035                 }
2036                 break;
2037         case MSR_FAM10H_MMIO_CONF_BASE:
2038                 if (data != 0) {
2039                         vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
2040                                     "0x%llx\n", data);
2041                         return 1;
2042                 }
2043                 break;
2044         case MSR_IA32_DEBUGCTLMSR:
2045                 if (!data) {
2046                         /* We support the non-activated case already */
2047                         break;
2048                 } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
2049                         /* Values other than LBR and BTF are vendor-specific,
2050                            thus reserved and should throw a #GP */
2051                         return 1;
2052                 }
2053                 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
2054                             __func__, data);
2055                 break;
2056         case 0x200 ... 0x2ff:
2057                 return set_msr_mtrr(vcpu, msr, data);
2058         case MSR_IA32_APICBASE:
2059                 return kvm_set_apic_base(vcpu, msr_info);
2060         case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2061                 return kvm_x2apic_msr_write(vcpu, msr, data);
2062         case MSR_IA32_TSCDEADLINE:
2063                 kvm_set_lapic_tscdeadline_msr(vcpu, data);
2064                 break;
2065         case MSR_IA32_TSC_ADJUST:
2066                 if (guest_cpuid_has_tsc_adjust(vcpu)) {
2067                         if (!msr_info->host_initiated) {
2068                                 u64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
2069                                 kvm_x86_ops->adjust_tsc_offset(vcpu, adj, true);
2070                         }
2071                         vcpu->arch.ia32_tsc_adjust_msr = data;
2072                 }
2073                 break;
2074         case MSR_IA32_MISC_ENABLE:
2075                 vcpu->arch.ia32_misc_enable_msr = data;
2076                 break;
2077         case MSR_KVM_WALL_CLOCK_NEW:
2078         case MSR_KVM_WALL_CLOCK:
2079                 vcpu->kvm->arch.wall_clock = data;
2080                 kvm_write_wall_clock(vcpu->kvm, data);
2081                 break;
2082         case MSR_KVM_SYSTEM_TIME_NEW:
2083         case MSR_KVM_SYSTEM_TIME: {
2084                 u64 gpa_offset;
2085                 kvmclock_reset(vcpu);
2086
2087                 vcpu->arch.time = data;
2088                 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2089
2090                 /* we verify if the enable bit is set... */
2091                 if (!(data & 1))
2092                         break;
2093
2094                 gpa_offset = data & ~(PAGE_MASK | 1);
2095
2096                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2097                      &vcpu->arch.pv_time, data & ~1ULL,
2098                      sizeof(struct pvclock_vcpu_time_info)))
2099                         vcpu->arch.pv_time_enabled = false;
2100                 else
2101                         vcpu->arch.pv_time_enabled = true;
2102
2103                 break;
2104         }
2105         case MSR_KVM_ASYNC_PF_EN:
2106                 if (kvm_pv_enable_async_pf(vcpu, data))
2107                         return 1;
2108                 break;
2109         case MSR_KVM_STEAL_TIME:
2110
2111                 if (unlikely(!sched_info_on()))
2112                         return 1;
2113
2114                 if (data & KVM_STEAL_RESERVED_MASK)
2115                         return 1;
2116
2117                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
2118                                                 data & KVM_STEAL_VALID_BITS,
2119                                                 sizeof(struct kvm_steal_time)))
2120                         return 1;
2121
2122                 vcpu->arch.st.msr_val = data;
2123
2124                 if (!(data & KVM_MSR_ENABLED))
2125                         break;
2126
2127                 vcpu->arch.st.last_steal = current->sched_info.run_delay;
2128
2129                 preempt_disable();
2130                 accumulate_steal_time(vcpu);
2131                 preempt_enable();
2132
2133                 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2134
2135                 break;
2136         case MSR_KVM_PV_EOI_EN:
2137                 if (kvm_lapic_enable_pv_eoi(vcpu, data))
2138                         return 1;
2139                 break;
2140
2141         case MSR_IA32_MCG_CTL:
2142         case MSR_IA32_MCG_STATUS:
2143         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
2144                 return set_msr_mce(vcpu, msr, data);
2145
2146         /* Performance counters are not protected by a CPUID bit,
2147          * so we should check all of them in the generic path for the sake of
2148          * cross vendor migration.
2149          * Writing a zero into the event select MSRs disables them,
2150          * which we perfectly emulate ;-). Any other value should be at least
2151          * reported, some guests depend on them.
2152          */
2153         case MSR_K7_EVNTSEL0:
2154         case MSR_K7_EVNTSEL1:
2155         case MSR_K7_EVNTSEL2:
2156         case MSR_K7_EVNTSEL3:
2157                 if (data != 0)
2158                         vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
2159                                     "0x%x data 0x%llx\n", msr, data);
2160                 break;
2161         /* at least RHEL 4 unconditionally writes to the perfctr registers,
2162          * so we ignore writes to make it happy.
2163          */
2164         case MSR_K7_PERFCTR0:
2165         case MSR_K7_PERFCTR1:
2166         case MSR_K7_PERFCTR2:
2167         case MSR_K7_PERFCTR3:
2168                 vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
2169                             "0x%x data 0x%llx\n", msr, data);
2170                 break;
2171         case MSR_P6_PERFCTR0:
2172         case MSR_P6_PERFCTR1:
2173                 pr = true;
2174         case MSR_P6_EVNTSEL0:
2175         case MSR_P6_EVNTSEL1:
2176                 if (kvm_pmu_msr(vcpu, msr))
2177                         return kvm_pmu_set_msr(vcpu, msr_info);
2178
2179                 if (pr || data != 0)
2180                         vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
2181                                     "0x%x data 0x%llx\n", msr, data);
2182                 break;
2183         case MSR_K7_CLK_CTL:
2184                 /*
2185                  * Ignore all writes to this no longer documented MSR.
2186                  * Writes are only relevant for old K7 processors,
2187                  * all pre-dating SVM, but a recommended workaround from
2188                  * AMD for these chips. It is possible to specify the
2189                  * affected processor models on the command line, hence
2190                  * the need to ignore the workaround.
2191                  */
2192                 break;
2193         case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2194                 if (kvm_hv_msr_partition_wide(msr)) {
2195                         int r;
2196                         mutex_lock(&vcpu->kvm->lock);
2197                         r = set_msr_hyperv_pw(vcpu, msr, data);
2198                         mutex_unlock(&vcpu->kvm->lock);
2199                         return r;
2200                 } else
2201                         return set_msr_hyperv(vcpu, msr, data);
2202                 break;
2203         case MSR_IA32_BBL_CR_CTL3:
2204                 /* Drop writes to this legacy MSR -- see rdmsr
2205                  * counterpart for further detail.
2206                  */
2207                 vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
2208                 break;
2209         case MSR_AMD64_OSVW_ID_LENGTH:
2210                 if (!guest_cpuid_has_osvw(vcpu))
2211                         return 1;
2212                 vcpu->arch.osvw.length = data;
2213                 break;
2214         case MSR_AMD64_OSVW_STATUS:
2215                 if (!guest_cpuid_has_osvw(vcpu))
2216                         return 1;
2217                 vcpu->arch.osvw.status = data;
2218                 break;
2219         default:
2220                 if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
2221                         return xen_hvm_config(vcpu, data);
2222                 if (kvm_pmu_msr(vcpu, msr))
2223                         return kvm_pmu_set_msr(vcpu, msr_info);
2224                 if (!ignore_msrs) {
2225                         vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
2226                                     msr, data);
2227                         return 1;
2228                 } else {
2229                         vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
2230                                     msr, data);
2231                         break;
2232                 }
2233         }
2234         return 0;
2235 }
2236 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
2237
2238
2239 /*
2240  * Reads an msr value (of 'msr_index') into 'pdata'.
2241  * Returns 0 on success, non-0 otherwise.
2242  * Assumes vcpu_load() was already called.
2243  */
2244 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
2245 {
2246         return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
2247 }
2248
2249 static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2250 {
2251         u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
2252
2253         if (!msr_mtrr_valid(msr))
2254                 return 1;
2255
2256         if (msr == MSR_MTRRdefType)
2257                 *pdata = vcpu->arch.mtrr_state.def_type +
2258                          (vcpu->arch.mtrr_state.enabled << 10);
2259         else if (msr == MSR_MTRRfix64K_00000)
2260                 *pdata = p[0];
2261         else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
2262                 *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
2263         else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
2264                 *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
2265         else if (msr == MSR_IA32_CR_PAT)
2266                 *pdata = vcpu->arch.pat;
2267         else {  /* Variable MTRRs */
2268                 int idx, is_mtrr_mask;
2269                 u64 *pt;
2270
2271                 idx = (msr - 0x200) / 2;
2272                 is_mtrr_mask = msr - 0x200 - 2 * idx;
2273                 if (!is_mtrr_mask)
2274                         pt =
2275                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
2276                 else
2277                         pt =
2278                           (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
2279                 *pdata = *pt;
2280         }
2281
2282         return 0;
2283 }
2284
2285 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2286 {
2287         u64 data;
2288         u64 mcg_cap = vcpu->arch.mcg_cap;
2289         unsigned bank_num = mcg_cap & 0xff;
2290
2291         switch (msr) {
2292         case MSR_IA32_P5_MC_ADDR:
2293         case MSR_IA32_P5_MC_TYPE:
2294                 data = 0;
2295                 break;
2296         case MSR_IA32_MCG_CAP:
2297                 data = vcpu->arch.mcg_cap;
2298                 break;
2299         case MSR_IA32_MCG_CTL:
2300                 if (!(mcg_cap & MCG_CTL_P))
2301                         return 1;
2302                 data = vcpu->arch.mcg_ctl;
2303                 break;
2304         case MSR_IA32_MCG_STATUS:
2305                 data = vcpu->arch.mcg_status;
2306                 break;
2307         default:
2308                 if (msr >= MSR_IA32_MC0_CTL &&
2309                     msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
2310                         u32 offset = msr - MSR_IA32_MC0_CTL;
2311                         data = vcpu->arch.mce_banks[offset];
2312                         break;
2313                 }
2314                 return 1;
2315         }
2316         *pdata = data;
2317         return 0;
2318 }
2319
2320 static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2321 {
2322         u64 data = 0;
2323         struct kvm *kvm = vcpu->kvm;
2324
2325         switch (msr) {
2326         case HV_X64_MSR_GUEST_OS_ID:
2327                 data = kvm->arch.hv_guest_os_id;
2328                 break;
2329         case HV_X64_MSR_HYPERCALL:
2330                 data = kvm->arch.hv_hypercall;
2331                 break;
2332         case HV_X64_MSR_TIME_REF_COUNT: {
2333                 data =
2334                      div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
2335                 break;
2336         }
2337         case HV_X64_MSR_REFERENCE_TSC:
2338                 data = kvm->arch.hv_tsc_page;
2339                 break;
2340         default:
2341                 vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2342                 return 1;
2343         }
2344
2345         *pdata = data;
2346         return 0;
2347 }
2348
2349 static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2350 {
2351         u64 data = 0;
2352
2353         switch (msr) {
2354         case HV_X64_MSR_VP_INDEX: {
2355                 int r;
2356                 struct kvm_vcpu *v;
2357                 kvm_for_each_vcpu(r, v, vcpu->kvm) {
2358                         if (v == vcpu) {
2359                                 data = r;
2360                                 break;
2361                         }
2362                 }
2363                 break;
2364         }
2365         case HV_X64_MSR_EOI:
2366                 return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
2367         case HV_X64_MSR_ICR:
2368                 return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
2369         case HV_X64_MSR_TPR:
2370                 return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
2371         case HV_X64_MSR_APIC_ASSIST_PAGE:
2372                 data = vcpu->arch.hv_vapic;
2373                 break;
2374         default:
2375                 vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
2376                 return 1;
2377         }
2378         *pdata = data;
2379         return 0;
2380 }
2381
2382 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
2383 {
2384         u64 data;
2385
2386         switch (msr) {
2387         case MSR_IA32_PLATFORM_ID:
2388         case MSR_IA32_EBL_CR_POWERON:
2389         case MSR_IA32_DEBUGCTLMSR:
2390         case MSR_IA32_LASTBRANCHFROMIP:
2391         case MSR_IA32_LASTBRANCHTOIP:
2392         case MSR_IA32_LASTINTFROMIP:
2393         case MSR_IA32_LASTINTTOIP:
2394         case MSR_K8_SYSCFG:
2395         case MSR_K7_HWCR:
2396         case MSR_VM_HSAVE_PA:
2397         case MSR_K7_EVNTSEL0:
2398         case MSR_K7_PERFCTR0:
2399         case MSR_K8_INT_PENDING_MSG:
2400         case MSR_AMD64_NB_CFG:
2401         case MSR_FAM10H_MMIO_CONF_BASE:
2402         case MSR_AMD64_BU_CFG2:
2403                 data = 0;
2404                 break;
2405         case MSR_P6_PERFCTR0:
2406         case MSR_P6_PERFCTR1:
2407         case MSR_P6_EVNTSEL0:
2408         case MSR_P6_EVNTSEL1:
2409                 if (kvm_pmu_msr(vcpu, msr))
2410                         return kvm_pmu_get_msr(vcpu, msr, pdata);
2411                 data = 0;
2412                 break;
2413         case MSR_IA32_UCODE_REV:
2414                 data = 0x100000000ULL;
2415                 break;
2416         case MSR_MTRRcap:
2417                 data = 0x500 | KVM_NR_VAR_MTRR;
2418                 break;
2419         case 0x200 ... 0x2ff:
2420                 return get_msr_mtrr(vcpu, msr, pdata);
2421         case 0xcd: /* fsb frequency */
2422                 data = 3;
2423                 break;
2424                 /*
2425                  * MSR_EBC_FREQUENCY_ID
2426                  * Conservative value valid for even the basic CPU models.
2427                  * Models 0,1: 000 in bits 23:21 indicating a bus speed of
2428                  * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
2429                  * and 266MHz for model 3, or 4. Set Core Clock
2430                  * Frequency to System Bus Frequency Ratio to 1 (bits
2431                  * 31:24) even though these are only valid for CPU
2432                  * models > 2, however guests may end up dividing or
2433                  * multiplying by zero otherwise.
2434                  */
2435         case MSR_EBC_FREQUENCY_ID:
2436                 data = 1 << 24;
2437                 break;
2438         case MSR_IA32_APICBASE:
2439                 data = kvm_get_apic_base(vcpu);
2440                 break;
2441         case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
2442                 return kvm_x2apic_msr_read(vcpu, msr, pdata);
2443                 break;
2444         case MSR_IA32_TSCDEADLINE:
2445                 data = kvm_get_lapic_tscdeadline_msr(vcpu);
2446                 break;
2447         case MSR_IA32_TSC_ADJUST:
2448                 data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
2449                 break;
2450         case MSR_IA32_MISC_ENABLE:
2451                 data = vcpu->arch.ia32_misc_enable_msr;
2452                 break;
2453         case MSR_IA32_PERF_STATUS:
2454                 /* TSC increment by tick */
2455                 data = 1000ULL;
2456                 /* CPU multiplier */
2457                 data |= (((uint64_t)4ULL) << 40);
2458                 break;
2459         case MSR_EFER:
2460                 data = vcpu->arch.efer;
2461                 break;
2462         case MSR_KVM_WALL_CLOCK:
2463         case MSR_KVM_WALL_CLOCK_NEW:
2464                 data = vcpu->kvm->arch.wall_clock;
2465                 break;
2466         case MSR_KVM_SYSTEM_TIME:
2467         case MSR_KVM_SYSTEM_TIME_NEW:
2468                 data = vcpu->arch.time;
2469                 break;
2470         case MSR_KVM_ASYNC_PF_EN:
2471                 data = vcpu->arch.apf.msr_val;
2472                 break;
2473         case MSR_KVM_STEAL_TIME:
2474                 data = vcpu->arch.st.msr_val;
2475                 break;
2476         case MSR_KVM_PV_EOI_EN:
2477                 data = vcpu->arch.pv_eoi.msr_val;
2478                 break;
2479         case MSR_IA32_P5_MC_ADDR:
2480         case MSR_IA32_P5_MC_TYPE:
2481         case MSR_IA32_MCG_CAP:
2482         case MSR_IA32_MCG_CTL:
2483         case MSR_IA32_MCG_STATUS:
2484         case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
2485                 return get_msr_mce(vcpu, msr, pdata);
2486         case MSR_K7_CLK_CTL:
2487                 /*
2488                  * Provide expected ramp-up count for K7. All other
2489                  * are set to zero, indicating minimum divisors for
2490                  * every field.
2491                  *
2492                  * This prevents guest kernels on AMD host with CPU
2493                  * type 6, model 8 and higher from exploding due to
2494                  * the rdmsr failing.
2495                  */
2496                 data = 0x20000000;
2497                 break;
2498         case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
2499                 if (kvm_hv_msr_partition_wide(msr)) {
2500                         int r;
2501                         mutex_lock(&vcpu->kvm->lock);
2502                         r = get_msr_hyperv_pw(vcpu, msr, pdata);
2503                         mutex_unlock(&vcpu->kvm->lock);
2504                         return r;
2505                 } else
2506                         return get_msr_hyperv(vcpu, msr, pdata);
2507                 break;
2508         case MSR_IA32_BBL_CR_CTL3:
2509                 /* This legacy MSR exists but isn't fully documented in current
2510                  * silicon.  It is however accessed by winxp in very narrow
2511                  * scenarios where it sets bit #19, itself documented as
2512                  * a "reserved" bit.  Best effort attempt to source coherent
2513                  * read data here should the balance of the register be
2514                  * interpreted by the guest:
2515                  *
2516                  * L2 cache control register 3: 64GB range, 256KB size,
2517                  * enabled, latency 0x1, configured
2518                  */
2519                 data = 0xbe702111;
2520                 break;
2521         case MSR_AMD64_OSVW_ID_LENGTH:
2522                 if (!guest_cpuid_has_osvw(vcpu))
2523                         return 1;
2524                 data = vcpu->arch.osvw.length;
2525                 break;
2526         case MSR_AMD64_OSVW_STATUS:
2527                 if (!guest_cpuid_has_osvw(vcpu))
2528                         return 1;
2529                 data = vcpu->arch.osvw.status;
2530                 break;
2531         default:
2532                 if (kvm_pmu_msr(vcpu, msr))
2533                         return kvm_pmu_get_msr(vcpu, msr, pdata);
2534                 if (!ignore_msrs) {
2535                         vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
2536                         return 1;
2537                 } else {
2538                         vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
2539                         data = 0;
2540                 }
2541                 break;
2542         }
2543         *pdata = data;
2544         return 0;
2545 }
2546 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
2547
2548 /*
2549  * Read or write a bunch of msrs. All parameters are kernel addresses.
2550  *
2551  * @return number of msrs set successfully.
2552  */
2553 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
2554                     struct kvm_msr_entry *entries,
2555                     int (*do_msr)(struct kvm_vcpu *vcpu,
2556                                   unsigned index, u64 *data))
2557 {
2558         int i, idx;
2559
2560         idx = srcu_read_lock(&vcpu->kvm->srcu);
2561         for (i = 0; i < msrs->nmsrs; ++i)
2562                 if (do_msr(vcpu, entries[i].index, &entries[i].data))
2563                         break;
2564         srcu_read_unlock(&vcpu->kvm->srcu, idx);
2565
2566         return i;
2567 }
2568
2569 /*
2570  * Read or write a bunch of msrs. Parameters are user addresses.
2571  *
2572  * @return number of msrs set successfully.
2573  */
2574 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
2575                   int (*do_msr)(struct kvm_vcpu *vcpu,
2576                                 unsigned index, u64 *data),
2577                   int writeback)
2578 {
2579         struct kvm_msrs msrs;
2580         struct kvm_msr_entry *entries;
2581         int r, n;
2582         unsigned size;
2583
2584         r = -EFAULT;
2585         if (copy_from_user(&msrs, user_msrs, sizeof msrs))
2586                 goto out;
2587
2588         r = -E2BIG;
2589         if (msrs.nmsrs >= MAX_IO_MSRS)
2590                 goto out;
2591
2592         size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2593         entries = memdup_user(user_msrs->entries, size);
2594         if (IS_ERR(entries)) {
2595                 r = PTR_ERR(entries);
2596                 goto out;
2597         }
2598
2599         r = n = __msr_io(vcpu, &msrs, entries, do_msr);
2600         if (r < 0)
2601                 goto out_free;
2602
2603         r = -EFAULT;
2604         if (writeback && copy_to_user(user_msrs->entries, entries, size))
2605                 goto out_free;
2606
2607         r = n;
2608
2609 out_free:
2610         kfree(entries);
2611 out:
2612         return r;
2613 }
2614
2615 int kvm_dev_ioctl_check_extension(long ext)
2616 {
2617         int r;
2618
2619         switch (ext) {
2620         case KVM_CAP_IRQCHIP:
2621         case KVM_CAP_HLT:
2622         case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
2623         case KVM_CAP_SET_TSS_ADDR:
2624         case KVM_CAP_EXT_CPUID:
2625         case KVM_CAP_EXT_EMUL_CPUID:
2626         case KVM_CAP_CLOCKSOURCE:
2627         case KVM_CAP_PIT:
2628         case KVM_CAP_NOP_IO_DELAY:
2629         case KVM_CAP_MP_STATE:
2630         case KVM_CAP_SYNC_MMU:
2631         case KVM_CAP_USER_NMI:
2632         case KVM_CAP_REINJECT_CONTROL:
2633         case KVM_CAP_IRQ_INJECT_STATUS:
2634         case KVM_CAP_IRQFD:
2635         case KVM_CAP_IOEVENTFD:
2636         case KVM_CAP_IOEVENTFD_NO_LENGTH:
2637         case KVM_CAP_PIT2:
2638         case KVM_CAP_PIT_STATE2:
2639         case KVM_CAP_SET_IDENTITY_MAP_ADDR:
2640         case KVM_CAP_XEN_HVM:
2641         case KVM_CAP_ADJUST_CLOCK:
2642         case KVM_CAP_VCPU_EVENTS:
2643         case KVM_CAP_HYPERV:
2644         case KVM_CAP_HYPERV_VAPIC:
2645         case KVM_CAP_HYPERV_SPIN:
2646         case KVM_CAP_PCI_SEGMENT:
2647         case KVM_CAP_DEBUGREGS:
2648         case KVM_CAP_X86_ROBUST_SINGLESTEP:
2649         case KVM_CAP_XSAVE:
2650         case KVM_CAP_ASYNC_PF:
2651         case KVM_CAP_GET_TSC_KHZ:
2652         case KVM_CAP_KVMCLOCK_CTRL:
2653         case KVM_CAP_READONLY_MEM:
2654         case KVM_CAP_HYPERV_TIME:
2655         case KVM_CAP_IOAPIC_POLARITY_IGNORED:
2656 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
2657         case KVM_CAP_ASSIGN_DEV_IRQ:
2658         case KVM_CAP_PCI_2_3:
2659 #endif
2660                 r = 1;
2661                 break;
2662         case KVM_CAP_COALESCED_MMIO:
2663                 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
2664                 break;
2665         case KVM_CAP_VAPIC:
2666                 r = !kvm_x86_ops->cpu_has_accelerated_tpr();
2667                 break;
2668         case KVM_CAP_NR_VCPUS:
2669                 r = KVM_SOFT_MAX_VCPUS;
2670                 break;
2671         case KVM_CAP_MAX_VCPUS:
2672                 r = KVM_MAX_VCPUS;
2673                 break;
2674         case KVM_CAP_NR_MEMSLOTS:
2675                 r = KVM_USER_MEM_SLOTS;
2676                 break;
2677         case KVM_CAP_PV_MMU:    /* obsolete */
2678                 r = 0;
2679                 break;
2680 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
2681         case KVM_CAP_IOMMU:
2682                 r = iommu_present(&pci_bus_type);
2683                 break;
2684 #endif
2685         case KVM_CAP_MCE:
2686                 r = KVM_MAX_MCE_BANKS;
2687                 break;
2688         case KVM_CAP_XCRS:
2689                 r = cpu_has_xsave;
2690                 break;
2691         case KVM_CAP_TSC_CONTROL:
2692                 r = kvm_has_tsc_control;
2693                 break;
2694         case KVM_CAP_TSC_DEADLINE_TIMER:
2695                 r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
2696                 break;
2697         default:
2698                 r = 0;
2699                 break;
2700         }
2701         return r;
2702
2703 }
2704
2705 long kvm_arch_dev_ioctl(struct file *filp,
2706                         unsigned int ioctl, unsigned long arg)
2707 {
2708         void __user *argp = (void __user *)arg;
2709         long r;
2710
2711         switch (ioctl) {
2712         case KVM_GET_MSR_INDEX_LIST: {
2713                 struct kvm_msr_list __user *user_msr_list = argp;
2714                 struct kvm_msr_list msr_list;
2715                 unsigned n;
2716
2717                 r = -EFAULT;
2718                 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
2719                         goto out;
2720                 n = msr_list.nmsrs;
2721                 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
2722                 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
2723                         goto out;
2724                 r = -E2BIG;
2725                 if (n < msr_list.nmsrs)
2726                         goto out;
2727                 r = -EFAULT;
2728                 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
2729                                  num_msrs_to_save * sizeof(u32)))
2730                         goto out;
2731                 if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
2732                                  &emulated_msrs,
2733                                  ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
2734                         goto out;
2735                 r = 0;
2736                 break;
2737         }
2738         case KVM_GET_SUPPORTED_CPUID:
2739         case KVM_GET_EMULATED_CPUID: {
2740                 struct kvm_cpuid2 __user *cpuid_arg = argp;
2741                 struct kvm_cpuid2 cpuid;
2742
2743                 r = -EFAULT;
2744                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
2745                         goto out;
2746
2747                 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
2748                                             ioctl);
2749                 if (r)
2750                         goto out;
2751
2752                 r = -EFAULT;
2753                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
2754                         goto out;
2755                 r = 0;
2756                 break;
2757         }
2758         case KVM_X86_GET_MCE_CAP_SUPPORTED: {
2759                 u64 mce_cap;
2760
2761                 mce_cap = KVM_MCE_CAP_SUPPORTED;
2762                 r = -EFAULT;
2763                 if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
2764                         goto out;
2765                 r = 0;
2766                 break;
2767         }
2768         default:
2769                 r = -EINVAL;
2770         }
2771 out:
2772         return r;
2773 }
2774
2775 static void wbinvd_ipi(void *garbage)
2776 {
2777         wbinvd();
2778 }
2779
2780 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
2781 {
2782         return kvm_arch_has_noncoherent_dma(vcpu->kvm);
2783 }
2784
2785 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2786 {
2787         /* Address WBINVD may be executed by guest */
2788         if (need_emulate_wbinvd(vcpu)) {
2789                 if (kvm_x86_ops->has_wbinvd_exit())
2790                         cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
2791                 else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
2792                         smp_call_function_single(vcpu->cpu,
2793                                         wbinvd_ipi, NULL, 1);
2794         }
2795
2796         kvm_x86_ops->vcpu_load(vcpu, cpu);
2797
2798         /* Apply any externally detected TSC adjustments (due to suspend) */
2799         if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
2800                 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
2801                 vcpu->arch.tsc_offset_adjustment = 0;
2802                 set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
2803         }
2804
2805         if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
2806                 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
2807                                 native_read_tsc() - vcpu->arch.last_host_tsc;
2808                 if (tsc_delta < 0)
2809                         mark_tsc_unstable("KVM discovered backwards TSC");
2810                 if (check_tsc_unstable()) {
2811                         u64 offset = kvm_x86_ops->compute_tsc_offset(vcpu,
2812                                                 vcpu->arch.last_guest_tsc);
2813                         kvm_x86_ops->write_tsc_offset(vcpu, offset);
2814                         vcpu->arch.tsc_catchup = 1;
2815                 }
2816                 /*
2817                  * On a host with synchronized TSC, there is no need to update
2818                  * kvmclock on vcpu->cpu migration
2819                  */
2820                 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
2821                         kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2822                 if (vcpu->cpu != cpu)
2823                         kvm_migrate_timers(vcpu);
2824                 vcpu->cpu = cpu;
2825         }
2826
2827         accumulate_steal_time(vcpu);
2828         kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
2829 }
2830
2831 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2832 {
2833         kvm_x86_ops->vcpu_put(vcpu);
2834         kvm_put_guest_fpu(vcpu);
2835         vcpu->arch.last_host_tsc = native_read_tsc();
2836 }
2837
2838 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
2839                                     struct kvm_lapic_state *s)
2840 {
2841         kvm_x86_ops->sync_pir_to_irr(vcpu);
2842         memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
2843
2844         return 0;
2845 }
2846
2847 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
2848                                     struct kvm_lapic_state *s)
2849 {
2850         kvm_apic_post_state_restore(vcpu, s);
2851         update_cr8_intercept(vcpu);
2852
2853         return 0;
2854 }
2855
2856 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2857                                     struct kvm_interrupt *irq)
2858 {
2859         if (irq->irq >= KVM_NR_INTERRUPTS)
2860                 return -EINVAL;
2861         if (irqchip_in_kernel(vcpu->kvm))
2862                 return -ENXIO;
2863
2864         kvm_queue_interrupt(vcpu, irq->irq, false);
2865         kvm_make_request(KVM_REQ_EVENT, vcpu);
2866
2867         return 0;
2868 }
2869
2870 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
2871 {
2872         kvm_inject_nmi(vcpu);
2873
2874         return 0;
2875 }
2876
2877 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
2878                                            struct kvm_tpr_access_ctl *tac)
2879 {
2880         if (tac->flags)
2881                 return -EINVAL;
2882         vcpu->arch.tpr_access_reporting = !!tac->enabled;
2883         return 0;
2884 }
2885
2886 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
2887                                         u64 mcg_cap)
2888 {
2889         int r;
2890         unsigned bank_num = mcg_cap & 0xff, bank;
2891
2892         r = -EINVAL;
2893         if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
2894                 goto out;
2895         if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
2896                 goto out;
2897         r = 0;
2898         vcpu->arch.mcg_cap = mcg_cap;
2899         /* Init IA32_MCG_CTL to all 1s */
2900         if (mcg_cap & MCG_CTL_P)
2901                 vcpu->arch.mcg_ctl = ~(u64)0;
2902         /* Init IA32_MCi_CTL to all 1s */
2903         for (bank = 0; bank < bank_num; bank++)
2904                 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
2905 out:
2906         return r;
2907 }
2908
2909 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
2910                                       struct kvm_x86_mce *mce)
2911 {
2912         u64 mcg_cap = vcpu->arch.mcg_cap;
2913         unsigned bank_num = mcg_cap & 0xff;
2914         u64 *banks = vcpu->arch.mce_banks;
2915
2916         if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
2917                 return -EINVAL;
2918         /*
2919          * if IA32_MCG_CTL is not all 1s, the uncorrected error
2920          * reporting is disabled
2921          */
2922         if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
2923             vcpu->arch.mcg_ctl != ~(u64)0)
2924                 return 0;
2925         banks += 4 * mce->bank;
2926         /*
2927          * if IA32_MCi_CTL is not all 1s, the uncorrected error
2928          * reporting is disabled for the bank
2929          */
2930         if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
2931                 return 0;
2932         if (mce->status & MCI_STATUS_UC) {
2933                 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
2934                     !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
2935                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2936                         return 0;
2937                 }
2938                 if (banks[1] & MCI_STATUS_VAL)
2939                         mce->status |= MCI_STATUS_OVER;
2940                 banks[2] = mce->addr;
2941                 banks[3] = mce->misc;
2942                 vcpu->arch.mcg_status = mce->mcg_status;
2943                 banks[1] = mce->status;
2944                 kvm_queue_exception(vcpu, MC_VECTOR);
2945         } else if (!(banks[1] & MCI_STATUS_VAL)
2946                    || !(banks[1] & MCI_STATUS_UC)) {
2947                 if (banks[1] & MCI_STATUS_VAL)
2948                         mce->status |= MCI_STATUS_OVER;
2949                 banks[2] = mce->addr;
2950                 banks[3] = mce->misc;
2951                 banks[1] = mce->status;
2952         } else
2953                 banks[1] |= MCI_STATUS_OVER;
2954         return 0;
2955 }
2956
2957 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
2958                                                struct kvm_vcpu_events *events)
2959 {
2960         process_nmi(vcpu);
2961         events->exception.injected =
2962                 vcpu->arch.exception.pending &&
2963                 !kvm_exception_is_soft(vcpu->arch.exception.nr);
2964         events->exception.nr = vcpu->arch.exception.nr;
2965         events->exception.has_error_code = vcpu->arch.exception.has_error_code;
2966         events->exception.pad = 0;
2967         events->exception.error_code = vcpu->arch.exception.error_code;
2968
2969         events->interrupt.injected =
2970                 vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
2971         events->interrupt.nr = vcpu->arch.interrupt.nr;
2972         events->interrupt.soft = 0;
2973         events->interrupt.shadow =
2974                 kvm_x86_ops->get_interrupt_shadow(vcpu,
2975                         KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
2976
2977         events->nmi.injected = vcpu->arch.nmi_injected;
2978         events->nmi.pending = vcpu->arch.nmi_pending != 0;
2979         events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
2980         events->nmi.pad = 0;
2981
2982         events->sipi_vector = 0; /* never valid when reporting to user space */
2983
2984         events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
2985                          | KVM_VCPUEVENT_VALID_SHADOW);
2986         memset(&events->reserved, 0, sizeof(events->reserved));
2987 }
2988
2989 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
2990                                               struct kvm_vcpu_events *events)
2991 {
2992         if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
2993                               | KVM_VCPUEVENT_VALID_SIPI_VECTOR
2994                               | KVM_VCPUEVENT_VALID_SHADOW))
2995                 return -EINVAL;
2996
2997         process_nmi(vcpu);
2998         vcpu->arch.exception.pending = events->exception.injected;
2999         vcpu->arch.exception.nr = events->exception.nr;
3000         vcpu->arch.exception.has_error_code = events->exception.has_error_code;
3001         vcpu->arch.exception.error_code = events->exception.error_code;
3002
3003         vcpu->arch.interrupt.pending = events->interrupt.injected;
3004         vcpu->arch.interrupt.nr = events->interrupt.nr;
3005         vcpu->arch.interrupt.soft = events->interrupt.soft;
3006         if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
3007                 kvm_x86_ops->set_interrupt_shadow(vcpu,
3008                                                   events->interrupt.shadow);
3009
3010         vcpu->arch.nmi_injected = events->nmi.injected;
3011         if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
3012                 vcpu->arch.nmi_pending = events->nmi.pending;
3013         kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
3014
3015         if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
3016             kvm_vcpu_has_lapic(vcpu))
3017                 vcpu->arch.apic->sipi_vector = events->sipi_vector;
3018
3019         kvm_make_request(KVM_REQ_EVENT, vcpu);
3020
3021         return 0;
3022 }
3023
3024 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
3025                                              struct kvm_debugregs *dbgregs)
3026 {
3027         unsigned long val;
3028
3029         memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
3030         _kvm_get_dr(vcpu, 6, &val);
3031         dbgregs->dr6 = val;
3032         dbgregs->dr7 = vcpu->arch.dr7;
3033         dbgregs->flags = 0;
3034         memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
3035 }
3036
3037 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
3038                                             struct kvm_debugregs *dbgregs)
3039 {
3040         if (dbgregs->flags)
3041                 return -EINVAL;
3042
3043         memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
3044         vcpu->arch.dr6 = dbgregs->dr6;
3045         kvm_update_dr6(vcpu);
3046         vcpu->arch.dr7 = dbgregs->dr7;
3047         kvm_update_dr7(vcpu);
3048
3049         return 0;
3050 }
3051
3052 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
3053                                          struct kvm_xsave *guest_xsave)
3054 {
3055         if (cpu_has_xsave) {
3056                 memcpy(guest_xsave->region,
3057                         &vcpu->arch.guest_fpu.state->xsave,
3058                         vcpu->arch.guest_xstate_size);
3059                 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] &=
3060                         vcpu->arch.guest_supported_xcr0 | XSTATE_FPSSE;
3061         } else {
3062                 memcpy(guest_xsave->region,
3063                         &vcpu->arch.guest_fpu.state->fxsave,
3064                         sizeof(struct i387_fxsave_struct));
3065                 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] =
3066                         XSTATE_FPSSE;
3067         }
3068 }
3069
3070 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
3071                                         struct kvm_xsave *guest_xsave)
3072 {
3073         u64 xstate_bv =
3074                 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)];
3075
3076         if (cpu_has_xsave) {
3077                 /*
3078                  * Here we allow setting states that are not present in
3079                  * CPUID leaf 0xD, index 0, EDX:EAX.  This is for compatibility
3080                  * with old userspace.
3081                  */
3082                 if (xstate_bv & ~kvm_supported_xcr0())
3083                         return -EINVAL;
3084                 memcpy(&vcpu->arch.guest_fpu.state->xsave,
3085                         guest_xsave->region, vcpu->arch.guest_xstate_size);
3086         } else {
3087                 if (xstate_bv & ~XSTATE_FPSSE)
3088                         return -EINVAL;
3089                 memcpy(&vcpu->arch.guest_fpu.state->fxsave,
3090                         guest_xsave->region, sizeof(struct i387_fxsave_struct));
3091         }
3092         return 0;
3093 }
3094
3095 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
3096                                         struct kvm_xcrs *guest_xcrs)
3097 {
3098         if (!cpu_has_xsave) {
3099                 guest_xcrs->nr_xcrs = 0;
3100                 return;
3101         }
3102
3103         guest_xcrs->nr_xcrs = 1;
3104         guest_xcrs->flags = 0;
3105         guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
3106         guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
3107 }
3108
3109 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
3110                                        struct kvm_xcrs *guest_xcrs)
3111 {
3112         int i, r = 0;
3113
3114         if (!cpu_has_xsave)
3115                 return -EINVAL;
3116
3117         if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
3118                 return -EINVAL;
3119
3120         for (i = 0; i < guest_xcrs->nr_xcrs; i++)
3121                 /* Only support XCR0 currently */
3122                 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
3123                         r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
3124                                 guest_xcrs->xcrs[i].value);
3125                         break;
3126                 }
3127         if (r)
3128                 r = -EINVAL;
3129         return r;
3130 }
3131
3132 /*
3133  * kvm_set_guest_paused() indicates to the guest kernel that it has been
3134  * stopped by the hypervisor.  This function will be called from the host only.
3135  * EINVAL is returned when the host attempts to set the flag for a guest that
3136  * does not support pv clocks.
3137  */
3138 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
3139 {
3140         if (!vcpu->arch.pv_time_enabled)
3141                 return -EINVAL;
3142         vcpu->arch.pvclock_set_guest_stopped_request = true;
3143         kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3144         return 0;
3145 }
3146
3147 long kvm_arch_vcpu_ioctl(struct file *filp,
3148                          unsigned int ioctl, unsigned long arg)
3149 {
3150         struct kvm_vcpu *vcpu = filp->private_data;
3151         void __user *argp = (void __user *)arg;
3152         int r;
3153         union {
3154                 struct kvm_lapic_state *lapic;
3155                 struct kvm_xsave *xsave;
3156                 struct kvm_xcrs *xcrs;
3157                 void *buffer;
3158         } u;
3159
3160         u.buffer = NULL;
3161         switch (ioctl) {
3162         case KVM_GET_LAPIC: {
3163                 r = -EINVAL;
3164                 if (!vcpu->arch.apic)
3165                         goto out;
3166                 u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
3167
3168                 r = -ENOMEM;
3169                 if (!u.lapic)
3170                         goto out;
3171                 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
3172                 if (r)
3173                         goto out;
3174                 r = -EFAULT;
3175                 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
3176                         goto out;
3177                 r = 0;
3178                 break;
3179         }
3180         case KVM_SET_LAPIC: {
3181                 r = -EINVAL;
3182                 if (!vcpu->arch.apic)
3183                         goto out;
3184                 u.lapic = memdup_user(argp, sizeof(*u.lapic));
3185                 if (IS_ERR(u.lapic))
3186                         return PTR_ERR(u.lapic);
3187
3188                 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
3189                 break;
3190         }
3191         case KVM_INTERRUPT: {
3192                 struct kvm_interrupt irq;
3193
3194                 r = -EFAULT;
3195                 if (copy_from_user(&irq, argp, sizeof irq))
3196                         goto out;
3197                 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
3198                 break;
3199         }
3200         case KVM_NMI: {
3201                 r = kvm_vcpu_ioctl_nmi(vcpu);
3202                 break;
3203         }
3204         case KVM_SET_CPUID: {
3205                 struct kvm_cpuid __user *cpuid_arg = argp;
3206                 struct kvm_cpuid cpuid;
3207
3208                 r = -EFAULT;
3209                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
3210                         goto out;
3211                 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
3212                 break;
3213         }
3214         case KVM_SET_CPUID2: {
3215                 struct kvm_cpuid2 __user *cpuid_arg = argp;
3216                 struct kvm_cpuid2 cpuid;
3217
3218                 r = -EFAULT;
3219                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
3220                         goto out;
3221                 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
3222                                               cpuid_arg->entries);
3223                 break;
3224         }
3225         case KVM_GET_CPUID2: {
3226                 struct kvm_cpuid2 __user *cpuid_arg = argp;
3227                 struct kvm_cpuid2 cpuid;
3228
3229                 r = -EFAULT;
3230                 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
3231                         goto out;
3232                 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
3233                                               cpuid_arg->entries);
3234                 if (r)
3235                         goto out;
3236                 r = -EFAULT;
3237                 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
3238                         goto out;
3239                 r = 0;
3240                 break;
3241         }
3242         case KVM_GET_MSRS:
3243                 r = msr_io(vcpu, argp, kvm_get_msr, 1);
3244                 break;
3245         case KVM_SET_MSRS:
3246                 r = msr_io(vcpu, argp, do_set_msr, 0);
3247                 break;
3248         case KVM_TPR_ACCESS_REPORTING: {
3249                 struct kvm_tpr_access_ctl tac;
3250
3251                 r = -EFAULT;
3252                 if (copy_from_user(&tac, argp, sizeof tac))
3253                         goto out;
3254                 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
3255                 if (r)
3256                         goto out;
3257                 r = -EFAULT;
3258                 if (copy_to_user(argp, &tac, sizeof tac))
3259                         goto out;
3260                 r = 0;
3261                 break;
3262         };
3263         case KVM_SET_VAPIC_ADDR: {
3264                 struct kvm_vapic_addr va;
3265
3266                 r = -EINVAL;
3267                 if (!irqchip_in_kernel(vcpu->kvm))
3268                         goto out;
3269                 r = -EFAULT;
3270                 if (copy_from_user(&va, argp, sizeof va))
3271                         goto out;
3272                 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
3273                 break;
3274         }
3275         case KVM_X86_SETUP_MCE: {
3276                 u64 mcg_cap;
3277
3278                 r = -EFAULT;
3279                 if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
3280                         goto out;
3281                 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
3282                 break;
3283         }
3284         case KVM_X86_SET_MCE: {
3285                 struct kvm_x86_mce mce;
3286
3287                 r = -EFAULT;
3288                 if (copy_from_user(&mce, argp, sizeof mce))
3289                         goto out;
3290                 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
3291                 break;
3292         }
3293         case KVM_GET_VCPU_EVENTS: {
3294                 struct kvm_vcpu_events events;
3295
3296                 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
3297
3298                 r = -EFAULT;
3299                 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
3300                         break;
3301                 r = 0;
3302                 break;
3303         }
3304         case KVM_SET_VCPU_EVENTS: {
3305                 struct kvm_vcpu_events events;
3306
3307                 r = -EFAULT;
3308                 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
3309                         break;
3310
3311                 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
3312                 break;
3313         }
3314         case KVM_GET_DEBUGREGS: {
3315                 struct kvm_debugregs dbgregs;
3316
3317                 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
3318
3319                 r = -EFAULT;
3320                 if (copy_to_user(argp, &dbgregs,
3321                                  sizeof(struct kvm_debugregs)))
3322                         break;
3323                 r = 0;
3324                 break;
3325         }
3326         case KVM_SET_DEBUGREGS: {
3327                 struct kvm_debugregs dbgregs;
3328
3329                 r = -EFAULT;
3330                 if (copy_from_user(&dbgregs, argp,
3331                                    sizeof(struct kvm_debugregs)))
3332                         break;
3333
3334                 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
3335                 break;
3336         }
3337         case KVM_GET_XSAVE: {
3338                 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL);
3339                 r = -ENOMEM;
3340                 if (!u.xsave)
3341                         break;
3342
3343                 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
3344
3345                 r = -EFAULT;
3346                 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
3347                         break;
3348                 r = 0;
3349                 break;
3350         }
3351         case KVM_SET_XSAVE: {
3352                 u.xsave = memdup_user(argp, sizeof(*u.xsave));
3353                 if (IS_ERR(u.xsave))
3354                         return PTR_ERR(u.xsave);
3355
3356                 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
3357                 break;
3358         }
3359         case KVM_GET_XCRS: {
3360                 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL);
3361                 r = -ENOMEM;
3362                 if (!u.xcrs)
3363                         break;
3364
3365                 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
3366
3367                 r = -EFAULT;
3368                 if (copy_to_user(argp, u.xcrs,
3369                                  sizeof(struct kvm_xcrs)))
3370                         break;
3371                 r = 0;
3372                 break;
3373         }
3374         case KVM_SET_XCRS: {
3375                 u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
3376                 if (IS_ERR(u.xcrs))
3377                         return PTR_ERR(u.xcrs);
3378
3379                 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
3380                 break;
3381         }
3382         case KVM_SET_TSC_KHZ: {
3383                 u32 user_tsc_khz;
3384
3385                 r = -EINVAL;
3386                 user_tsc_khz = (u32)arg;
3387
3388                 if (user_tsc_khz >= kvm_max_guest_tsc_khz)
3389                         goto out;
3390
3391                 if (user_tsc_khz == 0)
3392                         user_tsc_khz = tsc_khz;
3393
3394                 kvm_set_tsc_khz(vcpu, user_tsc_khz);
3395
3396                 r = 0;
3397                 goto out;
3398         }
3399         case KVM_GET_TSC_KHZ: {
3400                 r = vcpu->arch.virtual_tsc_khz;
3401                 goto out;
3402         }
3403         case KVM_KVMCLOCK_CTRL: {
3404                 r = kvm_set_guest_paused(vcpu);
3405                 goto out;
3406         }
3407         default:
3408                 r = -EINVAL;
3409         }
3410 out:
3411         kfree(u.buffer);
3412         return r;
3413 }
3414
3415 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3416 {
3417         return VM_FAULT_SIGBUS;
3418 }
3419
3420 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
3421 {
3422         int ret;
3423
3424         if (addr > (unsigned int)(-3 * PAGE_SIZE))
3425                 return -EINVAL;
3426         ret = kvm_x86_ops->set_tss_addr(kvm, addr);
3427         return ret;
3428 }
3429
3430 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
3431                                               u64 ident_addr)
3432 {
3433         kvm->arch.ept_identity_map_addr = ident_addr;
3434         return 0;
3435 }
3436
3437 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
3438                                           u32 kvm_nr_mmu_pages)
3439 {
3440         if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
3441                 return -EINVAL;
3442
3443         mutex_lock(&kvm->slots_lock);
3444
3445         kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
3446         kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
3447
3448         mutex_unlock(&kvm->slots_lock);
3449         return 0;
3450 }
3451
3452 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
3453 {
3454         return kvm->arch.n_max_mmu_pages;
3455 }
3456
3457 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
3458 {
3459         int r;
3460
3461         r = 0;
3462         switch (chip->chip_id) {
3463         case KVM_IRQCHIP_PIC_MASTER:
3464                 memcpy(&chip->chip.pic,
3465                         &pic_irqchip(kvm)->pics[0],
3466                         sizeof(struct kvm_pic_state));
3467                 break;
3468         case KVM_IRQCHIP_PIC_SLAVE:
3469                 memcpy(&chip->chip.pic,
3470                         &pic_irqchip(kvm)->pics[1],
3471                         sizeof(struct kvm_pic_state));
3472                 break;
3473         case KVM_IRQCHIP_IOAPIC:
3474                 r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
3475                 break;
3476         default:
3477                 r = -EINVAL;
3478                 break;
3479         }
3480         return r;
3481 }
3482
3483 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
3484 {
3485         int r;
3486
3487         r = 0;
3488         switch (chip->chip_id) {
3489         case KVM_IRQCHIP_PIC_MASTER:
3490                 spin_lock(&pic_irqchip(kvm)->lock);
3491                 memcpy(&pic_irqchip(kvm)->pics[0],
3492                         &chip->chip.pic,
3493                         sizeof(struct kvm_pic_state));
3494                 spin_unlock(&pic_irqchip(kvm)->lock);
3495                 break;
3496         case KVM_IRQCHIP_PIC_SLAVE:
3497                 spin_lock(&pic_irqchip(kvm)->lock);
3498                 memcpy(&pic_irqchip(kvm)->pics[1],
3499                         &chip->chip.pic,
3500                         sizeof(struct kvm_pic_state));
3501                 spin_unlock(&pic_irqchip(kvm)->lock);
3502                 break;
3503         case KVM_IRQCHIP_IOAPIC:
3504                 r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
3505                 break;
3506         default:
3507                 r = -EINVAL;
3508                 break;
3509         }
3510         kvm_pic_update_irq(pic_irqchip(kvm));
3511         return r;
3512 }
3513
3514 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
3515 {
3516         int r = 0;
3517
3518         mutex_lock(&kvm->arch.vpit->pit_state.lock);
3519         memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
3520         mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3521         return r;
3522 }
3523
3524 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
3525 {
3526         int r = 0;
3527
3528         mutex_lock(&kvm->arch.vpit->pit_state.lock);
3529         memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
3530         kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
3531         mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3532         return r;
3533 }
3534
3535 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
3536 {
3537         int r = 0;
3538
3539         mutex_lock(&kvm->arch.vpit->pit_state.lock);
3540         memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
3541                 sizeof(ps->channels));
3542         ps->flags = kvm->arch.vpit->pit_state.flags;
3543         mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3544         memset(&ps->reserved, 0, sizeof(ps->reserved));
3545         return r;
3546 }
3547
3548 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
3549 {
3550         int r = 0, start = 0;
3551         u32 prev_legacy, cur_legacy;
3552         mutex_lock(&kvm->arch.vpit->pit_state.lock);
3553         prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
3554         cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
3555         if (!prev_legacy && cur_legacy)
3556                 start = 1;
3557         memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
3558                sizeof(kvm->arch.vpit->pit_state.channels));
3559         kvm->arch.vpit->pit_state.flags = ps->flags;
3560         kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
3561         mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3562         return r;
3563 }
3564
3565 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
3566                                  struct kvm_reinject_control *control)
3567 {
3568         if (!kvm->arch.vpit)
3569                 return -ENXIO;
3570         mutex_lock(&kvm->arch.vpit->pit_state.lock);
3571         kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
3572         mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3573         return 0;
3574 }
3575
3576 /**
3577  * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
3578  * @kvm: kvm instance
3579  * @log: slot id and address to which we copy the log
3580  *
3581  * We need to keep it in mind that VCPU threads can write to the bitmap
3582  * concurrently.  So, to avoid losing data, we keep the following order for
3583  * each bit:
3584  *
3585  *   1. Take a snapshot of the bit and clear it if needed.
3586  *   2. Write protect the corresponding page.
3587  *   3. Flush TLB's if needed.
3588  *   4. Copy the snapshot to the userspace.
3589  *
3590  * Between 2 and 3, the guest may write to the page using the remaining TLB
3591  * entry.  This is not a problem because the page will be reported dirty at
3592  * step 4 using the snapshot taken before and step 3 ensures that successive
3593  * writes will be logged for the next call.
3594  */
3595 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
3596 {
3597         int r;
3598         struct kvm_memory_slot *memslot;
3599         unsigned long n, i;
3600         unsigned long *dirty_bitmap;
3601         unsigned long *dirty_bitmap_buffer;
3602         bool is_dirty = false;
3603
3604         mutex_lock(&kvm->slots_lock);
3605
3606         r = -EINVAL;
3607         if (log->slot >= KVM_USER_MEM_SLOTS)
3608                 goto out;
3609
3610         memslot = id_to_memslot(kvm->memslots, log->slot);
3611
3612         dirty_bitmap = memslot->dirty_bitmap;
3613         r = -ENOENT;
3614         if (!dirty_bitmap)
3615                 goto out;
3616
3617         n = kvm_dirty_bitmap_bytes(memslot);
3618
3619         dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
3620         memset(dirty_bitmap_buffer, 0, n);
3621
3622         spin_lock(&kvm->mmu_lock);
3623
3624         for (i = 0; i < n / sizeof(long); i++) {
3625                 unsigned long mask;
3626                 gfn_t offset;
3627
3628                 if (!dirty_bitmap[i])
3629                         continue;
3630
3631                 is_dirty = true;
3632
3633                 mask = xchg(&dirty_bitmap[i], 0);
3634                 dirty_bitmap_buffer[i] = mask;
3635
3636                 offset = i * BITS_PER_LONG;
3637                 kvm_mmu_write_protect_pt_masked(kvm, memslot, offset, mask);
3638         }
3639
3640         spin_unlock(&kvm->mmu_lock);
3641
3642         /* See the comments in kvm_mmu_slot_remove_write_access(). */
3643         lockdep_assert_held(&kvm->slots_lock);
3644
3645         /*
3646          * All the TLBs can be flushed out of mmu lock, see the comments in
3647          * kvm_mmu_slot_remove_write_access().
3648          */
3649         if (is_dirty)
3650                 kvm_flush_remote_tlbs(kvm);
3651
3652         r = -EFAULT;
3653         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
3654                 goto out;
3655
3656         r = 0;
3657 out:
3658         mutex_unlock(&kvm->slots_lock);
3659         return r;
3660 }
3661
3662 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
3663                         bool line_status)
3664 {
3665         if (!irqchip_in_kernel(kvm))
3666                 return -ENXIO;
3667
3668         irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
3669                                         irq_event->irq, irq_event->level,
3670                                         line_status);
3671         return 0;
3672 }
3673
3674 long kvm_arch_vm_ioctl(struct file *filp,
3675                        unsigned int ioctl, unsigned long arg)
3676 {
3677         struct kvm *kvm = filp->private_data;
3678         void __user *argp = (void __user *)arg;
3679         int r = -ENOTTY;
3680         /*
3681          * This union makes it completely explicit to gcc-3.x
3682          * that these two variables' stack usage should be
3683          * combined, not added together.
3684          */
3685         union {
3686                 struct kvm_pit_state ps;
3687                 struct kvm_pit_state2 ps2;
3688                 struct kvm_pit_config pit_config;
3689         } u;
3690
3691         switch (ioctl) {
3692         case KVM_SET_TSS_ADDR:
3693                 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
3694                 break;
3695         case KVM_SET_IDENTITY_MAP_ADDR: {
3696                 u64 ident_addr;
3697
3698                 r = -EFAULT;
3699                 if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
3700                         goto out;
3701                 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
3702                 break;
3703         }
3704         case KVM_SET_NR_MMU_PAGES:
3705                 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
3706                 break;
3707         case KVM_GET_NR_MMU_PAGES:
3708                 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
3709                 break;
3710         case KVM_CREATE_IRQCHIP: {
3711                 struct kvm_pic *vpic;
3712
3713                 mutex_lock(&kvm->lock);
3714                 r = -EEXIST;
3715                 if (kvm->arch.vpic)
3716                         goto create_irqchip_unlock;
3717                 r = -EINVAL;
3718                 if (atomic_read(&kvm->online_vcpus))
3719                         goto create_irqchip_unlock;
3720                 r = -ENOMEM;
3721                 vpic = kvm_create_pic(kvm);
3722                 if (vpic) {
3723                         r = kvm_ioapic_init(kvm);
3724                         if (r) {
3725                                 mutex_lock(&kvm->slots_lock);
3726                                 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3727                                                           &vpic->dev_master);
3728                                 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3729                                                           &vpic->dev_slave);
3730                                 kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
3731                                                           &vpic->dev_eclr);
3732                                 mutex_unlock(&kvm->slots_lock);
3733                                 kfree(vpic);
3734                                 goto create_irqchip_unlock;
3735                         }
3736                 } else
3737                         goto create_irqchip_unlock;
3738                 smp_wmb();
3739                 kvm->arch.vpic = vpic;
3740                 smp_wmb();
3741                 r = kvm_setup_default_irq_routing(kvm);
3742                 if (r) {
3743                         mutex_lock(&kvm->slots_lock);
3744                         mutex_lock(&kvm->irq_lock);
3745                         kvm_ioapic_destroy(kvm);
3746                         kvm_destroy_pic(kvm);
3747                         mutex_unlock(&kvm->irq_lock);
3748                         mutex_unlock(&kvm->slots_lock);
3749                 }
3750         create_irqchip_unlock:
3751                 mutex_unlock(&kvm->lock);
3752                 break;
3753         }
3754         case KVM_CREATE_PIT:
3755                 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
3756                 goto create_pit;
3757         case KVM_CREATE_PIT2:
3758                 r = -EFAULT;
3759                 if (copy_from_user(&u.pit_config, argp,
3760                                    sizeof(struct kvm_pit_config)))
3761                         goto out;
3762         create_pit:
3763                 mutex_lock(&kvm->slots_lock);
3764                 r = -EEXIST;
3765                 if (kvm->arch.vpit)
3766                         goto create_pit_unlock;
3767                 r = -ENOMEM;
3768                 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
3769                 if (kvm->arch.vpit)
3770                         r = 0;
3771         create_pit_unlock:
3772                 mutex_unlock(&kvm->slots_lock);
3773                 break;
3774         case KVM_GET_IRQCHIP: {
3775                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3776                 struct kvm_irqchip *chip;
3777
3778                 chip = memdup_user(argp, sizeof(*chip));
3779                 if (IS_ERR(chip)) {
3780                         r = PTR_ERR(chip);
3781                         goto out;
3782                 }
3783
3784                 r = -ENXIO;
3785                 if (!irqchip_in_kernel(kvm))
3786                         goto get_irqchip_out;
3787                 r = kvm_vm_ioctl_get_irqchip(kvm, chip);
3788                 if (r)
3789                         goto get_irqchip_out;
3790                 r = -EFAULT;
3791                 if (copy_to_user(argp, chip, sizeof *chip))
3792                         goto get_irqchip_out;
3793                 r = 0;
3794         get_irqchip_out:
3795                 kfree(chip);
3796                 break;
3797         }
3798         case KVM_SET_IRQCHIP: {
3799                 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3800                 struct kvm_irqchip *chip;
3801
3802                 chip = memdup_user(argp, sizeof(*chip));
3803                 if (IS_ERR(chip)) {
3804                         r = PTR_ERR(chip);
3805                         goto out;
3806                 }
3807
3808                 r = -ENXIO;
3809                 if (!irqchip_in_kernel(kvm))
3810                         goto set_irqchip_out;
3811                 r = kvm_vm_ioctl_set_irqchip(kvm, chip);
3812                 if (r)
3813                         goto set_irqchip_out;
3814                 r = 0;
3815         set_irqchip_out:
3816                 kfree(chip);
3817                 break;
3818         }
3819         case KVM_GET_PIT: {
3820                 r = -EFAULT;
3821                 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
3822                         goto out;
3823                 r = -ENXIO;
3824                 if (!kvm->arch.vpit)
3825                         goto out;
3826                 r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
3827                 if (r)
3828                         goto out;
3829                 r = -EFAULT;
3830                 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
3831                         goto out;
3832                 r = 0;
3833                 break;
3834         }
3835         case KVM_SET_PIT: {
3836                 r = -EFAULT;
3837                 if (copy_from_user(&u.ps, argp, sizeof u.ps))
3838                         goto out;
3839                 r = -ENXIO;
3840                 if (!kvm->arch.vpit)
3841                         goto out;
3842                 r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
3843                 break;
3844         }
3845         case KVM_GET_PIT2: {
3846                 r = -ENXIO;
3847                 if (!kvm->arch.vpit)
3848                         goto out;
3849                 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
3850                 if (r)
3851                         goto out;
3852                 r = -EFAULT;
3853                 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
3854                         goto out;
3855                 r = 0;
3856                 break;
3857         }
3858         case KVM_SET_PIT2: {
3859                 r = -EFAULT;
3860                 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
3861                         goto out;
3862                 r = -ENXIO;
3863                 if (!kvm->arch.vpit)
3864                         goto out;
3865                 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
3866                 break;
3867         }
3868         case KVM_REINJECT_CONTROL: {
3869                 struct kvm_reinject_control control;
3870                 r =  -EFAULT;
3871                 if (copy_from_user(&control, argp, sizeof(control)))
3872                         goto out;
3873                 r = kvm_vm_ioctl_reinject(kvm, &control);
3874                 break;
3875         }
3876         case KVM_XEN_HVM_CONFIG: {
3877                 r = -EFAULT;
3878                 if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
3879                                    sizeof(struct kvm_xen_hvm_config)))
3880                         goto out;
3881                 r = -EINVAL;
3882                 if (kvm->arch.xen_hvm_config.flags)
3883                         goto out;
3884                 r = 0;
3885                 break;
3886         }
3887         case KVM_SET_CLOCK: {
3888                 struct kvm_clock_data user_ns;
3889                 u64 now_ns;
3890                 s64 delta;
3891
3892                 r = -EFAULT;
3893                 if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
3894                         goto out;
3895
3896                 r = -EINVAL;
3897                 if (user_ns.flags)
3898                         goto out;
3899
3900                 r = 0;
3901                 local_irq_disable();
3902                 now_ns = get_kernel_ns();
3903                 delta = user_ns.clock - now_ns;
3904                 local_irq_enable();
3905                 kvm->arch.kvmclock_offset = delta;
3906                 kvm_gen_update_masterclock(kvm);
3907                 break;
3908         }
3909         case KVM_GET_CLOCK: {
3910                 struct kvm_clock_data user_ns;
3911                 u64 now_ns;
3912
3913                 local_irq_disable();
3914                 now_ns = get_kernel_ns();
3915                 user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
3916                 local_irq_enable();
3917                 user_ns.flags = 0;
3918                 memset(&user_ns.pad, 0, sizeof(user_ns.pad));
3919
3920                 r = -EFAULT;
3921                 if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
3922                         goto out;
3923                 r = 0;
3924                 break;
3925         }
3926
3927         default:
3928                 ;
3929         }
3930 out:
3931         return r;
3932 }
3933
3934 static void kvm_init_msr_list(void)
3935 {
3936         u32 dummy[2];
3937         unsigned i, j;
3938
3939         /* skip the first msrs in the list. KVM-specific */
3940         for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
3941                 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
3942                         continue;
3943
3944                 /*
3945                  * Even MSRs that are valid in the host may not be exposed
3946                  * to the guests in some cases.  We could work around this
3947                  * in VMX with the generic MSR save/load machinery, but it
3948                  * is not really worthwhile since it will really only
3949                  * happen with nested virtualization.
3950                  */
3951                 switch (msrs_to_save[i]) {
3952                 case MSR_IA32_BNDCFGS:
3953                         if (!kvm_x86_ops->mpx_supported())
3954                                 continue;
3955                         break;
3956                 default:
3957                         break;
3958                 }
3959
3960                 if (j < i)
3961                         msrs_to_save[j] = msrs_to_save[i];
3962                 j++;
3963         }
3964         num_msrs_to_save = j;
3965 }
3966
3967 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
3968                            const void *v)
3969 {
3970         int handled = 0;
3971         int n;
3972
3973         do {
3974                 n = min(len, 8);
3975                 if (!(vcpu->arch.apic &&
3976                       !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, n, v))
3977                     && kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
3978                         break;
3979                 handled += n;
3980                 addr += n;
3981                 len -= n;
3982                 v += n;
3983         } while (len);
3984
3985         return handled;
3986 }
3987
3988 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
3989 {
3990         int handled = 0;
3991         int n;
3992
3993         do {
3994                 n = min(len, 8);
3995                 if (!(vcpu->arch.apic &&
3996                       !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, n, v))
3997                     && kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, n, v))
3998                         break;
3999                 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v);
4000                 handled += n;
4001                 addr += n;
4002                 len -= n;
4003                 v += n;
4004         } while (len);
4005
4006         return handled;
4007 }
4008
4009 static void kvm_set_segment(struct kvm_vcpu *vcpu,
4010                         struct kvm_segment *var, int seg)
4011 {
4012         kvm_x86_ops->set_segment(vcpu, var, seg);
4013 }
4014
4015 void kvm_get_segment(struct kvm_vcpu *vcpu,
4016                      struct kvm_segment *var, int seg)
4017 {
4018         kvm_x86_ops->get_segment(vcpu, var, seg);
4019 }
4020
4021 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access)
4022 {
4023         gpa_t t_gpa;
4024         struct x86_exception exception;
4025
4026         BUG_ON(!mmu_is_nested(vcpu));
4027
4028         /* NPT walks are always user-walks */
4029         access |= PFERR_USER_MASK;
4030         t_gpa  = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, &exception);
4031
4032         return t_gpa;
4033 }
4034
4035 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
4036                               struct x86_exception *exception)
4037 {
4038         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4039         return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4040 }
4041
4042  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
4043                                 struct x86_exception *exception)
4044 {
4045         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4046         access |= PFERR_FETCH_MASK;
4047         return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4048 }
4049
4050 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
4051                                struct x86_exception *exception)
4052 {
4053         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4054         access |= PFERR_WRITE_MASK;
4055         return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4056 }
4057
4058 /* uses this to access any guest's mapped memory without checking CPL */
4059 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
4060                                 struct x86_exception *exception)
4061 {
4062         return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception);
4063 }
4064
4065 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
4066                                       struct kvm_vcpu *vcpu, u32 access,
4067                                       struct x86_exception *exception)
4068 {
4069         void *data = val;
4070         int r = X86EMUL_CONTINUE;
4071
4072         while (bytes) {
4073                 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access,
4074                                                             exception);
4075                 unsigned offset = addr & (PAGE_SIZE-1);
4076                 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
4077                 int ret;
4078
4079                 if (gpa == UNMAPPED_GVA)
4080                         return X86EMUL_PROPAGATE_FAULT;
4081                 ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
4082                 if (ret < 0) {
4083                         r = X86EMUL_IO_NEEDED;
4084                         goto out;
4085                 }
4086
4087                 bytes -= toread;
4088                 data += toread;
4089                 addr += toread;
4090         }
4091 out:
4092         return r;
4093 }
4094
4095 /* used for instruction fetching */
4096 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
4097                                 gva_t addr, void *val, unsigned int bytes,
4098                                 struct x86_exception *exception)
4099 {
4100         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4101         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4102
4103         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
4104                                           access | PFERR_FETCH_MASK,
4105                                           exception);
4106 }
4107
4108 int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
4109                                gva_t addr, void *val, unsigned int bytes,
4110                                struct x86_exception *exception)
4111 {
4112         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4113         u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
4114
4115         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
4116                                           exception);
4117 }
4118 EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
4119
4120 static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt,
4121                                       gva_t addr, void *val, unsigned int bytes,
4122                                       struct x86_exception *exception)
4123 {
4124         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4125         return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception);
4126 }
4127
4128 int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
4129                                        gva_t addr, void *val,
4130                                        unsigned int bytes,
4131                                        struct x86_exception *exception)
4132 {
4133         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4134         void *data = val;
4135         int r = X86EMUL_CONTINUE;
4136
4137         while (bytes) {
4138                 gpa_t gpa =  vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr,
4139                                                              PFERR_WRITE_MASK,
4140                                                              exception);
4141                 unsigned offset = addr & (PAGE_SIZE-1);
4142                 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
4143                 int ret;
4144
4145                 if (gpa == UNMAPPED_GVA)
4146                         return X86EMUL_PROPAGATE_FAULT;
4147                 ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
4148                 if (ret < 0) {
4149                         r = X86EMUL_IO_NEEDED;
4150                         goto out;
4151                 }
4152
4153                 bytes -= towrite;
4154                 data += towrite;
4155                 addr += towrite;
4156         }
4157 out:
4158         return r;
4159 }
4160 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
4161
4162 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
4163                                 gpa_t *gpa, struct x86_exception *exception,
4164                                 bool write)
4165 {
4166         u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
4167                 | (write ? PFERR_WRITE_MASK : 0);
4168
4169         if (vcpu_match_mmio_gva(vcpu, gva)
4170             && !permission_fault(vcpu, vcpu->arch.walk_mmu,
4171                                  vcpu->arch.access, access)) {
4172                 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
4173                                         (gva & (PAGE_SIZE - 1));
4174                 trace_vcpu_match_mmio(gva, *gpa, write, false);
4175                 return 1;
4176         }
4177
4178         *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
4179
4180         if (*gpa == UNMAPPED_GVA)
4181                 return -1;
4182
4183         /* For APIC access vmexit */
4184         if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
4185                 return 1;
4186
4187         if (vcpu_match_mmio_gpa(vcpu, *gpa)) {
4188                 trace_vcpu_match_mmio(gva, *gpa, write, true);
4189                 return 1;
4190         }
4191
4192         return 0;
4193 }
4194
4195 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
4196                         const void *val, int bytes)
4197 {
4198         int ret;
4199
4200         ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
4201         if (ret < 0)
4202                 return 0;
4203         kvm_mmu_pte_write(vcpu, gpa, val, bytes);
4204         return 1;
4205 }
4206
4207 struct read_write_emulator_ops {
4208         int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
4209                                   int bytes);
4210         int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
4211                                   void *val, int bytes);
4212         int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
4213                                int bytes, void *val);
4214         int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
4215                                     void *val, int bytes);
4216         bool write;
4217 };
4218
4219 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
4220 {
4221         if (vcpu->mmio_read_completed) {
4222                 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
4223                                vcpu->mmio_fragments[0].gpa, *(u64 *)val);
4224                 vcpu->mmio_read_completed = 0;
4225                 return 1;
4226         }
4227
4228         return 0;
4229 }
4230
4231 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
4232                         void *val, int bytes)
4233 {
4234         return !kvm_read_guest(vcpu->kvm, gpa, val, bytes);
4235 }
4236
4237 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
4238                          void *val, int bytes)
4239 {
4240         return emulator_write_phys(vcpu, gpa, val, bytes);
4241 }
4242
4243 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
4244 {
4245         trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
4246         return vcpu_mmio_write(vcpu, gpa, bytes, val);
4247 }
4248
4249 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
4250                           void *val, int bytes)
4251 {
4252         trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
4253         return X86EMUL_IO_NEEDED;
4254 }
4255
4256 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
4257                            void *val, int bytes)
4258 {
4259         struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
4260
4261         memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
4262         return X86EMUL_CONTINUE;
4263 }
4264
4265 static const struct read_write_emulator_ops read_emultor = {
4266         .read_write_prepare = read_prepare,
4267         .read_write_emulate = read_emulate,
4268         .read_write_mmio = vcpu_mmio_read,
4269         .read_write_exit_mmio = read_exit_mmio,
4270 };
4271
4272 static const struct read_write_emulator_ops write_emultor = {
4273         .read_write_emulate = write_emulate,
4274         .read_write_mmio = write_mmio,
4275         .read_write_exit_mmio = write_exit_mmio,
4276         .write = true,
4277 };
4278
4279 static int emulator_read_write_onepage(unsigned long addr, void *val,
4280                                        unsigned int bytes,
4281                                        struct x86_exception *exception,
4282                                        struct kvm_vcpu *vcpu,
4283                                        const struct read_write_emulator_ops *ops)
4284 {
4285         gpa_t gpa;
4286         int handled, ret;
4287         bool write = ops->write;
4288         struct kvm_mmio_fragment *frag;
4289
4290         ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
4291
4292         if (ret < 0)
4293                 return X86EMUL_PROPAGATE_FAULT;
4294
4295         /* For APIC access vmexit */
4296         if (ret)
4297                 goto mmio;
4298
4299         if (ops->read_write_emulate(vcpu, gpa, val, bytes))
4300                 return X86EMUL_CONTINUE;
4301
4302 mmio:
4303         /*
4304          * Is this MMIO handled locally?
4305          */
4306         handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
4307         if (handled == bytes)
4308                 return X86EMUL_CONTINUE;
4309
4310         gpa += handled;
4311         bytes -= handled;
4312         val += handled;
4313
4314         WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
4315         frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
4316         frag->gpa = gpa;
4317         frag->data = val;
4318         frag->len = bytes;
4319         return X86EMUL_CONTINUE;
4320 }
4321
4322 int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
4323                         void *val, unsigned int bytes,
4324                         struct x86_exception *exception,
4325                         const struct read_write_emulator_ops *ops)
4326 {
4327         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4328         gpa_t gpa;
4329         int rc;
4330
4331         if (ops->read_write_prepare &&
4332                   ops->read_write_prepare(vcpu, val, bytes))
4333                 return X86EMUL_CONTINUE;
4334
4335         vcpu->mmio_nr_fragments = 0;
4336
4337         /* Crossing a page boundary? */
4338         if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
4339                 int now;
4340
4341                 now = -addr & ~PAGE_MASK;
4342                 rc = emulator_read_write_onepage(addr, val, now, exception,
4343                                                  vcpu, ops);
4344
4345                 if (rc != X86EMUL_CONTINUE)
4346                         return rc;
4347                 addr += now;
4348                 val += now;
4349                 bytes -= now;
4350         }
4351
4352         rc = emulator_read_write_onepage(addr, val, bytes, exception,
4353                                          vcpu, ops);
4354         if (rc != X86EMUL_CONTINUE)
4355                 return rc;
4356
4357         if (!vcpu->mmio_nr_fragments)
4358                 return rc;
4359
4360         gpa = vcpu->mmio_fragments[0].gpa;
4361
4362         vcpu->mmio_needed = 1;
4363         vcpu->mmio_cur_fragment = 0;
4364
4365         vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
4366         vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
4367         vcpu->run->exit_reason = KVM_EXIT_MMIO;
4368         vcpu->run->mmio.phys_addr = gpa;
4369
4370         return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
4371 }
4372
4373 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
4374                                   unsigned long addr,
4375                                   void *val,
4376                                   unsigned int bytes,
4377                                   struct x86_exception *exception)
4378 {
4379         return emulator_read_write(ctxt, addr, val, bytes,
4380                                    exception, &read_emultor);
4381 }
4382
4383 int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
4384                             unsigned long addr,
4385                             const void *val,
4386                             unsigned int bytes,
4387                             struct x86_exception *exception)
4388 {
4389         return emulator_read_write(ctxt, addr, (void *)val, bytes,
4390                                    exception, &write_emultor);
4391 }
4392
4393 #define CMPXCHG_TYPE(t, ptr, old, new) \
4394         (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
4395
4396 #ifdef CONFIG_X86_64
4397 #  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
4398 #else
4399 #  define CMPXCHG64(ptr, old, new) \
4400         (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
4401 #endif
4402
4403 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
4404                                      unsigned long addr,
4405                                      const void *old,
4406                                      const void *new,
4407                                      unsigned int bytes,
4408                                      struct x86_exception *exception)
4409 {
4410         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4411         gpa_t gpa;
4412         struct page *page;
4413         char *kaddr;
4414         bool exchanged;
4415
4416         /* guests cmpxchg8b have to be emulated atomically */
4417         if (bytes > 8 || (bytes & (bytes - 1)))
4418                 goto emul_write;
4419
4420         gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
4421
4422         if (gpa == UNMAPPED_GVA ||
4423             (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
4424                 goto emul_write;
4425
4426         if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
4427                 goto emul_write;
4428
4429         page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
4430         if (is_error_page(page))
4431                 goto emul_write;
4432
4433         kaddr = kmap_atomic(page);
4434         kaddr += offset_in_page(gpa);
4435         switch (bytes) {
4436         case 1:
4437                 exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
4438                 break;
4439         case 2:
4440                 exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
4441                 break;
4442         case 4:
4443                 exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
4444                 break;
4445         case 8:
4446                 exchanged = CMPXCHG64(kaddr, old, new);
4447                 break;
4448         default:
4449                 BUG();
4450         }
4451         kunmap_atomic(kaddr);
4452         kvm_release_page_dirty(page);
4453
4454         if (!exchanged)
4455                 return X86EMUL_CMPXCHG_FAILED;
4456
4457         mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
4458         kvm_mmu_pte_write(vcpu, gpa, new, bytes);
4459
4460         return X86EMUL_CONTINUE;
4461
4462 emul_write:
4463         printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
4464
4465         return emulator_write_emulated(ctxt, addr, new, bytes, exception);
4466 }
4467
4468 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
4469 {
4470         /* TODO: String I/O for in kernel device */
4471         int r;
4472
4473         if (vcpu->arch.pio.in)
4474                 r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
4475                                     vcpu->arch.pio.size, pd);
4476         else
4477                 r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
4478                                      vcpu->arch.pio.port, vcpu->arch.pio.size,
4479                                      pd);
4480         return r;
4481 }
4482
4483 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
4484                                unsigned short port, void *val,
4485                                unsigned int count, bool in)
4486 {
4487         vcpu->arch.pio.port = port;
4488         vcpu->arch.pio.in = in;
4489         vcpu->arch.pio.count  = count;
4490         vcpu->arch.pio.size = size;
4491
4492         if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
4493                 vcpu->arch.pio.count = 0;
4494                 return 1;
4495         }
4496
4497         vcpu->run->exit_reason = KVM_EXIT_IO;
4498         vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
4499         vcpu->run->io.size = size;
4500         vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
4501         vcpu->run->io.count = count;
4502         vcpu->run->io.port = port;
4503
4504         return 0;
4505 }
4506
4507 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
4508                                     int size, unsigned short port, void *val,
4509                                     unsigned int count)
4510 {
4511         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4512         int ret;
4513
4514         if (vcpu->arch.pio.count)
4515                 goto data_avail;
4516
4517         ret = emulator_pio_in_out(vcpu, size, port, val, count, true);
4518         if (ret) {
4519 data_avail:
4520                 memcpy(val, vcpu->arch.pio_data, size * count);
4521                 trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data);
4522                 vcpu->arch.pio.count = 0;
4523                 return 1;
4524         }
4525
4526         return 0;
4527 }
4528
4529 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
4530                                      int size, unsigned short port,
4531                                      const void *val, unsigned int count)
4532 {
4533         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4534
4535         memcpy(vcpu->arch.pio_data, val, size * count);
4536         trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
4537         return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
4538 }
4539
4540 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
4541 {
4542         return kvm_x86_ops->get_segment_base(vcpu, seg);
4543 }
4544
4545 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
4546 {
4547         kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
4548 }
4549
4550 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
4551 {
4552         if (!need_emulate_wbinvd(vcpu))
4553                 return X86EMUL_CONTINUE;
4554
4555         if (kvm_x86_ops->has_wbinvd_exit()) {
4556                 int cpu = get_cpu();
4557
4558                 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4559                 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask,
4560                                 wbinvd_ipi, NULL, 1);
4561                 put_cpu();
4562                 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
4563         } else
4564                 wbinvd();
4565         return X86EMUL_CONTINUE;
4566 }
4567 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
4568
4569 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
4570 {
4571         kvm_emulate_wbinvd(emul_to_vcpu(ctxt));
4572 }
4573
4574 int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest)
4575 {
4576         return _kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
4577 }
4578
4579 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
4580 {
4581
4582         return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
4583 }
4584
4585 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
4586 {
4587         return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
4588 }
4589
4590 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
4591 {
4592         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4593         unsigned long value;
4594
4595         switch (cr) {
4596         case 0:
4597                 value = kvm_read_cr0(vcpu);
4598                 break;
4599         case 2:
4600                 value = vcpu->arch.cr2;
4601                 break;
4602         case 3:
4603                 value = kvm_read_cr3(vcpu);
4604                 break;
4605         case 4:
4606                 value = kvm_read_cr4(vcpu);
4607                 break;
4608         case 8:
4609                 value = kvm_get_cr8(vcpu);
4610                 break;
4611         default:
4612                 kvm_err("%s: unexpected cr %u\n", __func__, cr);
4613                 return 0;
4614         }
4615
4616         return value;
4617 }
4618
4619 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
4620 {
4621         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4622         int res = 0;
4623
4624         switch (cr) {
4625         case 0:
4626                 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
4627                 break;
4628         case 2:
4629                 vcpu->arch.cr2 = val;
4630                 break;
4631         case 3:
4632                 res = kvm_set_cr3(vcpu, val);
4633                 break;
4634         case 4:
4635                 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
4636                 break;
4637         case 8:
4638                 res = kvm_set_cr8(vcpu, val);
4639                 break;
4640         default:
4641                 kvm_err("%s: unexpected cr %u\n", __func__, cr);
4642                 res = -1;
4643         }
4644
4645         return res;
4646 }
4647
4648 static void emulator_set_rflags(struct x86_emulate_ctxt *ctxt, ulong val)
4649 {
4650         kvm_set_rflags(emul_to_vcpu(ctxt), val);
4651 }
4652
4653 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
4654 {
4655         return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt));
4656 }
4657
4658 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4659 {
4660         kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt);
4661 }
4662
4663 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4664 {
4665         kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt);
4666 }
4667
4668 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4669 {
4670         kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt);
4671 }
4672
4673 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
4674 {
4675         kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt);
4676 }
4677
4678 static unsigned long emulator_get_cached_segment_base(
4679         struct x86_emulate_ctxt *ctxt, int seg)
4680 {
4681         return get_segment_base(emul_to_vcpu(ctxt), seg);
4682 }
4683
4684 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
4685                                  struct desc_struct *desc, u32 *base3,
4686                                  int seg)
4687 {
4688         struct kvm_segment var;
4689
4690         kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
4691         *selector = var.selector;
4692
4693         if (var.unusable) {
4694                 memset(desc, 0, sizeof(*desc));
4695                 return false;
4696         }
4697
4698         if (var.g)
4699                 var.limit >>= 12;
4700         set_desc_limit(desc, var.limit);
4701         set_desc_base(desc, (unsigned long)var.base);
4702 #ifdef CONFIG_X86_64
4703         if (base3)
4704                 *base3 = var.base >> 32;
4705 #endif
4706         desc->type = var.type;
4707         desc->s = var.s;
4708         desc->dpl = var.dpl;
4709         desc->p = var.present;
4710         desc->avl = var.avl;
4711         desc->l = var.l;
4712         desc->d = var.db;
4713         desc->g = var.g;
4714
4715         return true;
4716 }
4717
4718 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
4719                                  struct desc_struct *desc, u32 base3,
4720                                  int seg)
4721 {
4722         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
4723         struct kvm_segment var;
4724
4725         var.selector = selector;
4726         var.base = get_desc_base(desc);
4727 #ifdef CONFIG_X86_64
4728         var.base |= ((u64)base3) << 32;
4729 #endif
4730         var.limit = get_desc_limit(desc);
4731         if (desc->g)
4732                 var.limit = (var.limit << 12) | 0xfff;
4733         var.type = desc->type;
4734         var.present = desc->p;
4735         var.dpl = desc->dpl;
4736         var.db = desc->d;
4737         var.s = desc->s;
4738         var.l = desc->l;
4739         var.g = desc->g;
4740         var.avl = desc->avl;
4741         var.present = desc->p;
4742         var.unusable = !var.present;
4743         var.padding = 0;
4744
4745         kvm_set_segment(vcpu, &var, seg);
4746         return;
4747 }
4748
4749 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
4750                             u32 msr_index, u64 *pdata)
4751 {
4752         return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
4753 }
4754
4755 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
4756                             u32 msr_index, u64 data)
4757 {
4758         struct msr_data msr;
4759
4760         msr.data = data;
4761         msr.index = msr_index;
4762         msr.host_initiated = false;
4763         return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
4764 }
4765
4766 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
4767                              u32 pmc, u64 *pdata)
4768 {
4769         return kvm_pmu_read_pmc(emul_to_vcpu(ctxt), pmc, pdata);
4770 }
4771
4772 static void emulator_halt(struct x86_emulate_ctxt *ctxt)
4773 {
4774         emul_to_vcpu(ctxt)->arch.halt_request = 1;
4775 }
4776
4777 static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt)
4778 {
4779         preempt_disable();
4780         kvm_load_guest_fpu(emul_to_vcpu(ctxt));
4781         /*
4782          * CR0.TS may reference the host fpu state, not the guest fpu state,
4783          * so it may be clear at this point.
4784          */
4785         clts();
4786 }
4787
4788 static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt)
4789 {
4790         preempt_enable();
4791 }
4792
4793 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
4794                               struct x86_instruction_info *info,
4795                               enum x86_intercept_stage stage)
4796 {
4797         return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage);
4798 }
4799
4800 static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
4801                                u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
4802 {
4803         kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
4804 }
4805
4806 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
4807 {
4808         return kvm_register_read(emul_to_vcpu(ctxt), reg);
4809 }
4810
4811 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
4812 {
4813         kvm_register_write(emul_to_vcpu(ctxt), reg, val);
4814 }
4815
4816 static const struct x86_emulate_ops emulate_ops = {
4817         .read_gpr            = emulator_read_gpr,
4818         .write_gpr           = emulator_write_gpr,
4819         .read_std            = kvm_read_guest_virt_system,
4820         .write_std           = kvm_write_guest_virt_system,
4821         .fetch               = kvm_fetch_guest_virt,
4822         .read_emulated       = emulator_read_emulated,
4823         .write_emulated      = emulator_write_emulated,
4824         .cmpxchg_emulated    = emulator_cmpxchg_emulated,
4825         .invlpg              = emulator_invlpg,
4826         .pio_in_emulated     = emulator_pio_in_emulated,
4827         .pio_out_emulated    = emulator_pio_out_emulated,
4828         .get_segment         = emulator_get_segment,
4829         .set_segment         = emulator_set_segment,
4830         .get_cached_segment_base = emulator_get_cached_segment_base,
4831         .get_gdt             = emulator_get_gdt,
4832         .get_idt             = emulator_get_idt,
4833         .set_gdt             = emulator_set_gdt,
4834         .set_idt             = emulator_set_idt,
4835         .get_cr              = emulator_get_cr,
4836         .set_cr              = emulator_set_cr,
4837         .set_rflags          = emulator_set_rflags,
4838         .cpl                 = emulator_get_cpl,
4839         .get_dr              = emulator_get_dr,
4840         .set_dr              = emulator_set_dr,
4841         .set_msr             = emulator_set_msr,
4842         .get_msr             = emulator_get_msr,
4843         .read_pmc            = emulator_read_pmc,
4844         .halt                = emulator_halt,
4845         .wbinvd              = emulator_wbinvd,
4846         .fix_hypercall       = emulator_fix_hypercall,
4847         .get_fpu             = emulator_get_fpu,
4848         .put_fpu             = emulator_put_fpu,
4849         .intercept           = emulator_intercept,
4850         .get_cpuid           = emulator_get_cpuid,
4851 };
4852
4853 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
4854 {
4855         u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
4856         /*
4857          * an sti; sti; sequence only disable interrupts for the first
4858          * instruction. So, if the last instruction, be it emulated or
4859          * not, left the system with the INT_STI flag enabled, it
4860          * means that the last instruction is an sti. We should not
4861          * leave the flag on in this case. The same goes for mov ss
4862          */
4863         if (!(int_shadow & mask))
4864                 kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
4865 }
4866
4867 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
4868 {
4869         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4870         if (ctxt->exception.vector == PF_VECTOR)
4871                 kvm_propagate_fault(vcpu, &ctxt->exception);
4872         else if (ctxt->exception.error_code_valid)
4873                 kvm_queue_exception_e(vcpu, ctxt->exception.vector,
4874                                       ctxt->exception.error_code);
4875         else
4876                 kvm_queue_exception(vcpu, ctxt->exception.vector);
4877 }
4878
4879 static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4880 {
4881         memset(&ctxt->opcode_len, 0,
4882                (void *)&ctxt->_regs - (void *)&ctxt->opcode_len);
4883
4884         ctxt->fetch.start = 0;
4885         ctxt->fetch.end = 0;
4886         ctxt->io_read.pos = 0;
4887         ctxt->io_read.end = 0;
4888         ctxt->mem_read.pos = 0;
4889         ctxt->mem_read.end = 0;
4890 }
4891
4892 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
4893 {
4894         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4895         int cs_db, cs_l;
4896
4897         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
4898
4899         ctxt->eflags = kvm_get_rflags(vcpu);
4900         ctxt->eip = kvm_rip_read(vcpu);
4901         ctxt->mode = (!is_protmode(vcpu))               ? X86EMUL_MODE_REAL :
4902                      (ctxt->eflags & X86_EFLAGS_VM)     ? X86EMUL_MODE_VM86 :
4903                      (cs_l && is_long_mode(vcpu))       ? X86EMUL_MODE_PROT64 :
4904                      cs_db                              ? X86EMUL_MODE_PROT32 :
4905                                                           X86EMUL_MODE_PROT16;
4906         ctxt->guest_mode = is_guest_mode(vcpu);
4907
4908         init_decode_cache(ctxt);
4909         vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4910 }
4911
4912 int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
4913 {
4914         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4915         int ret;
4916
4917         init_emulate_ctxt(vcpu);
4918
4919         ctxt->op_bytes = 2;
4920         ctxt->ad_bytes = 2;
4921         ctxt->_eip = ctxt->eip + inc_eip;
4922         ret = emulate_int_real(ctxt, irq);
4923
4924         if (ret != X86EMUL_CONTINUE)
4925                 return EMULATE_FAIL;
4926
4927         ctxt->eip = ctxt->_eip;
4928         kvm_rip_write(vcpu, ctxt->eip);
4929         kvm_set_rflags(vcpu, ctxt->eflags);
4930
4931         if (irq == NMI_VECTOR)
4932                 vcpu->arch.nmi_pending = 0;
4933         else
4934                 vcpu->arch.interrupt.pending = false;
4935
4936         return EMULATE_DONE;
4937 }
4938 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
4939
4940 static int handle_emulation_failure(struct kvm_vcpu *vcpu)
4941 {
4942         int r = EMULATE_DONE;
4943
4944         ++vcpu->stat.insn_emulation_fail;
4945         trace_kvm_emulate_insn_failed(vcpu);
4946         if (!is_guest_mode(vcpu)) {
4947                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
4948                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
4949                 vcpu->run->internal.ndata = 0;
4950                 r = EMULATE_FAIL;
4951         }
4952         kvm_queue_exception(vcpu, UD_VECTOR);
4953
4954         return r;
4955 }
4956
4957 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
4958                                   bool write_fault_to_shadow_pgtable,
4959                                   int emulation_type)
4960 {
4961         gpa_t gpa = cr2;
4962         pfn_t pfn;
4963
4964         if (emulation_type & EMULTYPE_NO_REEXECUTE)
4965                 return false;
4966
4967         if (!vcpu->arch.mmu.direct_map) {
4968                 /*
4969                  * Write permission should be allowed since only
4970                  * write access need to be emulated.
4971                  */
4972                 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
4973
4974                 /*
4975                  * If the mapping is invalid in guest, let cpu retry
4976                  * it to generate fault.
4977                  */
4978                 if (gpa == UNMAPPED_GVA)
4979                         return true;
4980         }
4981
4982         /*
4983          * Do not retry the unhandleable instruction if it faults on the
4984          * readonly host memory, otherwise it will goto a infinite loop:
4985          * retry instruction -> write #PF -> emulation fail -> retry
4986          * instruction -> ...
4987          */
4988         pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
4989
4990         /*
4991          * If the instruction failed on the error pfn, it can not be fixed,
4992          * report the error to userspace.
4993          */
4994         if (is_error_noslot_pfn(pfn))
4995                 return false;
4996
4997         kvm_release_pfn_clean(pfn);
4998
4999         /* The instructions are well-emulated on direct mmu. */
5000         if (vcpu->arch.mmu.direct_map) {
5001                 unsigned int indirect_shadow_pages;
5002
5003                 spin_lock(&vcpu->kvm->mmu_lock);
5004                 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
5005                 spin_unlock(&vcpu->kvm->mmu_lock);
5006
5007                 if (indirect_shadow_pages)
5008                         kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
5009
5010                 return true;
5011         }
5012
5013         /*
5014          * if emulation was due to access to shadowed page table
5015          * and it failed try to unshadow page and re-enter the
5016          * guest to let CPU execute the instruction.
5017          */
5018         kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
5019
5020         /*
5021          * If the access faults on its page table, it can not
5022          * be fixed by unprotecting shadow page and it should
5023          * be reported to userspace.
5024          */
5025         return !write_fault_to_shadow_pgtable;
5026 }
5027
5028 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
5029                               unsigned long cr2,  int emulation_type)
5030 {
5031         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5032         unsigned long last_retry_eip, last_retry_addr, gpa = cr2;
5033
5034         last_retry_eip = vcpu->arch.last_retry_eip;
5035         last_retry_addr = vcpu->arch.last_retry_addr;
5036
5037         /*
5038          * If the emulation is caused by #PF and it is non-page_table
5039          * writing instruction, it means the VM-EXIT is caused by shadow
5040          * page protected, we can zap the shadow page and retry this
5041          * instruction directly.
5042          *
5043          * Note: if the guest uses a non-page-table modifying instruction
5044          * on the PDE that points to the instruction, then we will unmap
5045          * the instruction and go to an infinite loop. So, we cache the
5046          * last retried eip and the last fault address, if we meet the eip
5047          * and the address again, we can break out of the potential infinite
5048          * loop.
5049          */
5050         vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
5051
5052         if (!(emulation_type & EMULTYPE_RETRY))
5053                 return false;
5054
5055         if (x86_page_table_writing_insn(ctxt))
5056                 return false;
5057
5058         if (ctxt->eip == last_retry_eip && last_retry_addr == cr2)
5059                 return false;
5060
5061         vcpu->arch.last_retry_eip = ctxt->eip;
5062         vcpu->arch.last_retry_addr = cr2;
5063
5064         if (!vcpu->arch.mmu.direct_map)
5065                 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL);
5066
5067         kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
5068
5069         return true;
5070 }
5071
5072 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
5073 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
5074
5075 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
5076                                 unsigned long *db)
5077 {
5078         u32 dr6 = 0;
5079         int i;
5080         u32 enable, rwlen;
5081
5082         enable = dr7;
5083         rwlen = dr7 >> 16;
5084         for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
5085                 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
5086                         dr6 |= (1 << i);
5087         return dr6;
5088 }
5089
5090 static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, int *r)
5091 {
5092         struct kvm_run *kvm_run = vcpu->run;
5093
5094         /*
5095          * Use the "raw" value to see if TF was passed to the processor.
5096          * Note that the new value of the flags has not been saved yet.
5097          *
5098          * This is correct even for TF set by the guest, because "the
5099          * processor will not generate this exception after the instruction
5100          * that sets the TF flag".
5101          */
5102         unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
5103
5104         if (unlikely(rflags & X86_EFLAGS_TF)) {
5105                 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
5106                         kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1;
5107                         kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip;
5108                         kvm_run->debug.arch.exception = DB_VECTOR;
5109                         kvm_run->exit_reason = KVM_EXIT_DEBUG;
5110                         *r = EMULATE_USER_EXIT;
5111                 } else {
5112                         vcpu->arch.emulate_ctxt.eflags &= ~X86_EFLAGS_TF;
5113                         /*
5114                          * "Certain debug exceptions may clear bit 0-3.  The
5115                          * remaining contents of the DR6 register are never
5116                          * cleared by the processor".
5117                          */
5118                         vcpu->arch.dr6 &= ~15;
5119                         vcpu->arch.dr6 |= DR6_BS;
5120                         kvm_queue_exception(vcpu, DB_VECTOR);
5121                 }
5122         }
5123 }
5124
5125 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
5126 {
5127         struct kvm_run *kvm_run = vcpu->run;
5128         unsigned long eip = vcpu->arch.emulate_ctxt.eip;
5129         u32 dr6 = 0;
5130
5131         if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
5132             (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
5133                 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
5134                                            vcpu->arch.guest_debug_dr7,
5135                                            vcpu->arch.eff_db);
5136
5137                 if (dr6 != 0) {
5138                         kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
5139                         kvm_run->debug.arch.pc = kvm_rip_read(vcpu) +
5140                                 get_segment_base(vcpu, VCPU_SREG_CS);
5141
5142                         kvm_run->debug.arch.exception = DB_VECTOR;
5143                         kvm_run->exit_reason = KVM_EXIT_DEBUG;
5144                         *r = EMULATE_USER_EXIT;
5145                         return true;
5146                 }
5147         }
5148
5149         if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK)) {
5150                 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
5151                                            vcpu->arch.dr7,
5152                                            vcpu->arch.db);
5153
5154                 if (dr6 != 0) {
5155                         vcpu->arch.dr6 &= ~15;
5156                         vcpu->arch.dr6 |= dr6;
5157                         kvm_queue_exception(vcpu, DB_VECTOR);
5158                         *r = EMULATE_DONE;
5159                         return true;
5160                 }
5161         }
5162
5163         return false;
5164 }
5165
5166 int x86_emulate_instruction(struct kvm_vcpu *vcpu,
5167                             unsigned long cr2,
5168                             int emulation_type,
5169                             void *insn,
5170                             int insn_len)
5171 {
5172         int r;
5173         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
5174         bool writeback = true;
5175         bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
5176
5177         /*
5178          * Clear write_fault_to_shadow_pgtable here to ensure it is
5179          * never reused.
5180          */
5181         vcpu->arch.write_fault_to_shadow_pgtable = false;
5182         kvm_clear_exception_queue(vcpu);
5183
5184         if (!(emulation_type & EMULTYPE_NO_DECODE)) {
5185                 init_emulate_ctxt(vcpu);
5186
5187                 /*
5188                  * We will reenter on the same instruction since
5189                  * we do not set complete_userspace_io.  This does not
5190                  * handle watchpoints yet, those would be handled in
5191                  * the emulate_ops.
5192                  */
5193                 if (kvm_vcpu_check_breakpoint(vcpu, &r))
5194                         return r;
5195
5196                 ctxt->interruptibility = 0;
5197                 ctxt->have_exception = false;
5198                 ctxt->perm_ok = false;
5199
5200                 ctxt->ud = emulation_type & EMULTYPE_TRAP_UD;
5201
5202                 r = x86_decode_insn(ctxt, insn, insn_len);
5203
5204                 trace_kvm_emulate_insn_start(vcpu);
5205                 ++vcpu->stat.insn_emulation;
5206                 if (r != EMULATION_OK)  {
5207                         if (emulation_type & EMULTYPE_TRAP_UD)
5208                                 return EMULATE_FAIL;
5209                         if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
5210                                                 emulation_type))
5211                                 return EMULATE_DONE;
5212                         if (emulation_type & EMULTYPE_SKIP)
5213                                 return EMULATE_FAIL;
5214                         return handle_emulation_failure(vcpu);
5215                 }
5216         }
5217
5218         if (emulation_type & EMULTYPE_SKIP) {
5219                 kvm_rip_write(vcpu, ctxt->_eip);
5220                 return EMULATE_DONE;
5221         }
5222
5223         if (retry_instruction(ctxt, cr2, emulation_type))
5224                 return EMULATE_DONE;
5225
5226         /* this is needed for vmware backdoor interface to work since it
5227            changes registers values  during IO operation */
5228         if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
5229                 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
5230                 emulator_invalidate_register_cache(ctxt);
5231         }
5232
5233 restart:
5234         r = x86_emulate_insn(ctxt);
5235
5236         if (r == EMULATION_INTERCEPTED)
5237                 return EMULATE_DONE;
5238
5239         if (r == EMULATION_FAILED) {
5240                 if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
5241                                         emulation_type))
5242                         return EMULATE_DONE;
5243
5244                 return handle_emulation_failure(vcpu);
5245         }
5246
5247         if (ctxt->have_exception) {
5248                 inject_emulated_exception(vcpu);
5249                 r = EMULATE_DONE;
5250         } else if (vcpu->arch.pio.count) {
5251                 if (!vcpu->arch.pio.in) {
5252                         /* FIXME: return into emulator if single-stepping.  */
5253                         vcpu->arch.pio.count = 0;
5254                 } else {
5255                         writeback = false;
5256                         vcpu->arch.complete_userspace_io = complete_emulated_pio;
5257                 }
5258                 r = EMULATE_USER_EXIT;
5259         } else if (vcpu->mmio_needed) {
5260                 if (!vcpu->mmio_is_write)
5261                         writeback = false;
5262                 r = EMULATE_USER_EXIT;
5263                 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
5264         } else if (r == EMULATION_RESTART)
5265                 goto restart;
5266         else
5267                 r = EMULATE_DONE;
5268
5269         if (writeback) {
5270                 toggle_interruptibility(vcpu, ctxt->interruptibility);
5271                 kvm_make_request(KVM_REQ_EVENT, vcpu);
5272                 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
5273                 kvm_rip_write(vcpu, ctxt->eip);
5274                 if (r == EMULATE_DONE)
5275                         kvm_vcpu_check_singlestep(vcpu, &r);
5276                 kvm_set_rflags(vcpu, ctxt->eflags);
5277         } else
5278                 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
5279
5280         return r;
5281 }
5282 EXPORT_SYMBOL_GPL(x86_emulate_instruction);
5283
5284 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
5285 {
5286         unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
5287         int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt,
5288                                             size, port, &val, 1);
5289         /* do not return to emulator after return from userspace */
5290         vcpu->arch.pio.count = 0;
5291         return ret;
5292 }
5293 EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
5294
5295 static void tsc_bad(void *info)
5296 {
5297         __this_cpu_write(cpu_tsc_khz, 0);
5298 }
5299
5300 static void tsc_khz_changed(void *data)
5301 {
5302         struct cpufreq_freqs *freq = data;
5303         unsigned long khz = 0;
5304
5305         if (data)
5306                 khz = freq->new;
5307         else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
5308                 khz = cpufreq_quick_get(raw_smp_processor_id());
5309         if (!khz)
5310                 khz = tsc_khz;
5311         __this_cpu_write(cpu_tsc_khz, khz);
5312 }
5313
5314 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
5315                                      void *data)
5316 {
5317         struct cpufreq_freqs *freq = data;
5318         struct kvm *kvm;
5319         struct kvm_vcpu *vcpu;
5320         int i, send_ipi = 0;
5321
5322         /*
5323          * We allow guests to temporarily run on slowing clocks,
5324          * provided we notify them after, or to run on accelerating
5325          * clocks, provided we notify them before.  Thus time never
5326          * goes backwards.
5327          *
5328          * However, we have a problem.  We can't atomically update
5329          * the frequency of a given CPU from this function; it is
5330          * merely a notifier, which can be called from any CPU.
5331          * Changing the TSC frequency at arbitrary points in time
5332          * requires a recomputation of local variables related to
5333          * the TSC for each VCPU.  We must flag these local variables
5334          * to be updated and be sure the update takes place with the
5335          * new frequency before any guests proceed.
5336          *
5337          * Unfortunately, the combination of hotplug CPU and frequency
5338          * change creates an intractable locking scenario; the order
5339          * of when these callouts happen is undefined with respect to
5340          * CPU hotplug, and they can race with each other.  As such,
5341          * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
5342          * undefined; you can actually have a CPU frequency change take
5343          * place in between the computation of X and the setting of the
5344          * variable.  To protect against this problem, all updates of
5345          * the per_cpu tsc_khz variable are done in an interrupt
5346          * protected IPI, and all callers wishing to update the value
5347          * must wait for a synchronous IPI to complete (which is trivial
5348          * if the caller is on the CPU already).  This establishes the
5349          * necessary total order on variable updates.
5350          *
5351          * Note that because a guest time update may take place
5352          * anytime after the setting of the VCPU's request bit, the
5353          * correct TSC value must be set before the request.  However,
5354          * to ensure the update actually makes it to any guest which
5355          * starts running in hardware virtualization between the set
5356          * and the acquisition of the spinlock, we must also ping the
5357          * CPU after setting the request bit.
5358          *
5359          */
5360
5361         if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
5362                 return 0;
5363         if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
5364                 return 0;
5365
5366         smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5367
5368         spin_lock(&kvm_lock);
5369         list_for_each_entry(kvm, &vm_list, vm_list) {
5370                 kvm_for_each_vcpu(i, vcpu, kvm) {
5371                         if (vcpu->cpu != freq->cpu)
5372                                 continue;
5373                         kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5374                         if (vcpu->cpu != smp_processor_id())
5375                                 send_ipi = 1;
5376                 }
5377         }
5378         spin_unlock(&kvm_lock);
5379
5380         if (freq->old < freq->new && send_ipi) {
5381                 /*
5382                  * We upscale the frequency.  Must make the guest
5383                  * doesn't see old kvmclock values while running with
5384                  * the new frequency, otherwise we risk the guest sees
5385                  * time go backwards.
5386                  *
5387                  * In case we update the frequency for another cpu
5388                  * (which might be in guest context) send an interrupt
5389                  * to kick the cpu out of guest context.  Next time
5390                  * guest context is entered kvmclock will be updated,
5391                  * so the guest will not see stale values.
5392                  */
5393                 smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1);
5394         }
5395         return 0;
5396 }
5397
5398 static struct notifier_block kvmclock_cpufreq_notifier_block = {
5399         .notifier_call  = kvmclock_cpufreq_notifier
5400 };
5401
5402 static int kvmclock_cpu_notifier(struct notifier_block *nfb,
5403                                         unsigned long action, void *hcpu)
5404 {
5405         unsigned int cpu = (unsigned long)hcpu;
5406
5407         switch (action) {
5408                 case CPU_ONLINE:
5409                 case CPU_DOWN_FAILED:
5410                         smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
5411                         break;
5412                 case CPU_DOWN_PREPARE:
5413                         smp_call_function_single(cpu, tsc_bad, NULL, 1);
5414                         break;
5415         }
5416         return NOTIFY_OK;
5417 }
5418
5419 static struct notifier_block kvmclock_cpu_notifier_block = {
5420         .notifier_call  = kvmclock_cpu_notifier,
5421         .priority = -INT_MAX
5422 };
5423
5424 static void kvm_timer_init(void)
5425 {
5426         int cpu;
5427
5428         max_tsc_khz = tsc_khz;
5429
5430         cpu_notifier_register_begin();
5431         if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
5432 #ifdef CONFIG_CPU_FREQ
5433                 struct cpufreq_policy policy;
5434                 memset(&policy, 0, sizeof(policy));
5435                 cpu = get_cpu();
5436                 cpufreq_get_policy(&policy, cpu);
5437                 if (policy.cpuinfo.max_freq)
5438                         max_tsc_khz = policy.cpuinfo.max_freq;
5439                 put_cpu();
5440 #endif
5441                 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
5442                                           CPUFREQ_TRANSITION_NOTIFIER);
5443         }
5444         pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
5445         for_each_online_cpu(cpu)
5446                 smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
5447
5448         __register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
5449         cpu_notifier_register_done();
5450
5451 }
5452
5453 static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
5454
5455 int kvm_is_in_guest(void)
5456 {
5457         return __this_cpu_read(current_vcpu) != NULL;
5458 }
5459
5460 static int kvm_is_user_mode(void)
5461 {
5462         int user_mode = 3;
5463
5464         if (__this_cpu_read(current_vcpu))
5465                 user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
5466
5467         return user_mode != 0;
5468 }
5469
5470 static unsigned long kvm_get_guest_ip(void)
5471 {
5472         unsigned long ip = 0;
5473
5474         if (__this_cpu_read(current_vcpu))
5475                 ip = kvm_rip_read(__this_cpu_read(current_vcpu));
5476
5477         return ip;
5478 }
5479
5480 static struct perf_guest_info_callbacks kvm_guest_cbs = {
5481         .is_in_guest            = kvm_is_in_guest,
5482         .is_user_mode           = kvm_is_user_mode,
5483         .get_guest_ip           = kvm_get_guest_ip,
5484 };
5485
5486 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
5487 {
5488         __this_cpu_write(current_vcpu, vcpu);
5489 }
5490 EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
5491
5492 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
5493 {
5494         __this_cpu_write(current_vcpu, NULL);
5495 }
5496 EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
5497
5498 static void kvm_set_mmio_spte_mask(void)
5499 {
5500         u64 mask;
5501         int maxphyaddr = boot_cpu_data.x86_phys_bits;
5502
5503         /*
5504          * Set the reserved bits and the present bit of an paging-structure
5505          * entry to generate page fault with PFER.RSV = 1.
5506          */
5507          /* Mask the reserved physical address bits. */
5508         mask = ((1ull << (51 - maxphyaddr + 1)) - 1) << maxphyaddr;
5509
5510         /* Bit 62 is always reserved for 32bit host. */
5511         mask |= 0x3ull << 62;
5512
5513         /* Set the present bit. */
5514         mask |= 1ull;
5515
5516 #ifdef CONFIG_X86_64
5517         /*
5518          * If reserved bit is not supported, clear the present bit to disable
5519          * mmio page fault.
5520          */
5521         if (maxphyaddr == 52)
5522                 mask &= ~1ull;
5523 #endif
5524
5525         kvm_mmu_set_mmio_spte_mask(mask);
5526 }
5527
5528 #ifdef CONFIG_X86_64
5529 static void pvclock_gtod_update_fn(struct work_struct *work)
5530 {
5531         struct kvm *kvm;
5532
5533         struct kvm_vcpu *vcpu;
5534         int i;
5535
5536         spin_lock(&kvm_lock);
5537         list_for_each_entry(kvm, &vm_list, vm_list)
5538                 kvm_for_each_vcpu(i, vcpu, kvm)
5539                         set_bit(KVM_REQ_MASTERCLOCK_UPDATE, &vcpu->requests);
5540         atomic_set(&kvm_guest_has_master_clock, 0);
5541         spin_unlock(&kvm_lock);
5542 }
5543
5544 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
5545
5546 /*
5547  * Notification about pvclock gtod data update.
5548  */
5549 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
5550                                void *priv)
5551 {
5552         struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
5553         struct timekeeper *tk = priv;
5554
5555         update_pvclock_gtod(tk);
5556
5557         /* disable master clock if host does not trust, or does not
5558          * use, TSC clocksource
5559          */
5560         if (gtod->clock.vclock_mode != VCLOCK_TSC &&
5561             atomic_read(&kvm_guest_has_master_clock) != 0)
5562                 queue_work(system_long_wq, &pvclock_gtod_work);
5563
5564         return 0;
5565 }
5566
5567 static struct notifier_block pvclock_gtod_notifier = {
5568         .notifier_call = pvclock_gtod_notify,
5569 };
5570 #endif
5571
5572 int kvm_arch_init(void *opaque)
5573 {
5574         int r;
5575         struct kvm_x86_ops *ops = opaque;
5576
5577         if (kvm_x86_ops) {
5578                 printk(KERN_ERR "kvm: already loaded the other module\n");
5579                 r = -EEXIST;
5580                 goto out;
5581         }
5582
5583         if (!ops->cpu_has_kvm_support()) {
5584                 printk(KERN_ERR "kvm: no hardware support\n");
5585                 r = -EOPNOTSUPP;
5586                 goto out;
5587         }
5588         if (ops->disabled_by_bios()) {
5589                 printk(KERN_ERR "kvm: disabled by bios\n");
5590                 r = -EOPNOTSUPP;
5591                 goto out;
5592         }
5593
5594         r = -ENOMEM;
5595         shared_msrs = alloc_percpu(struct kvm_shared_msrs);
5596         if (!shared_msrs) {
5597                 printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n");
5598                 goto out;
5599         }
5600
5601         r = kvm_mmu_module_init();
5602         if (r)
5603                 goto out_free_percpu;
5604
5605         kvm_set_mmio_spte_mask();
5606
5607         kvm_x86_ops = ops;
5608         kvm_init_msr_list();
5609
5610         kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
5611                         PT_DIRTY_MASK, PT64_NX_MASK, 0);
5612
5613         kvm_timer_init();
5614
5615         perf_register_guest_info_callbacks(&kvm_guest_cbs);
5616
5617         if (cpu_has_xsave)
5618                 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
5619
5620         kvm_lapic_init();
5621 #ifdef CONFIG_X86_64
5622         pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
5623 #endif
5624
5625         return 0;
5626
5627 out_free_percpu:
5628         free_percpu(shared_msrs);
5629 out:
5630         return r;
5631 }
5632
5633 void kvm_arch_exit(void)
5634 {
5635         perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
5636
5637         if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
5638                 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
5639                                             CPUFREQ_TRANSITION_NOTIFIER);
5640         unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
5641 #ifdef CONFIG_X86_64
5642         pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
5643 #endif
5644         kvm_x86_ops = NULL;
5645         kvm_mmu_module_exit();
5646         free_percpu(shared_msrs);
5647 }
5648
5649 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
5650 {
5651         ++vcpu->stat.halt_exits;
5652         if (irqchip_in_kernel(vcpu->kvm)) {
5653                 vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
5654                 return 1;
5655         } else {
5656                 vcpu->run->exit_reason = KVM_EXIT_HLT;
5657                 return 0;
5658         }
5659 }
5660 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
5661
5662 int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
5663 {
5664         u64 param, ingpa, outgpa, ret;
5665         uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
5666         bool fast, longmode;
5667         int cs_db, cs_l;
5668
5669         /*
5670          * hypercall generates UD from non zero cpl and real mode
5671          * per HYPER-V spec
5672          */
5673         if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
5674                 kvm_queue_exception(vcpu, UD_VECTOR);
5675                 return 0;
5676         }
5677
5678         kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
5679         longmode = is_long_mode(vcpu) && cs_l == 1;
5680
5681         if (!longmode) {
5682                 param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
5683                         (kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
5684                 ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
5685                         (kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
5686                 outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
5687                         (kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
5688         }
5689 #ifdef CONFIG_X86_64
5690         else {
5691                 param = kvm_register_read(vcpu, VCPU_REGS_RCX);
5692                 ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
5693                 outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
5694         }
5695 #endif
5696
5697         code = param & 0xffff;
5698         fast = (param >> 16) & 0x1;
5699         rep_cnt = (param >> 32) & 0xfff;
5700         rep_idx = (param >> 48) & 0xfff;
5701
5702         trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
5703
5704         switch (code) {
5705         case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
5706                 kvm_vcpu_on_spin(vcpu);
5707                 break;
5708         default:
5709                 res = HV_STATUS_INVALID_HYPERCALL_CODE;
5710                 break;
5711         }
5712
5713         ret = res | (((u64)rep_done & 0xfff) << 32);
5714         if (longmode) {
5715                 kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
5716         } else {
5717                 kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
5718                 kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
5719         }
5720
5721         return 1;
5722 }
5723
5724 /*
5725  * kvm_pv_kick_cpu_op:  Kick a vcpu.
5726  *
5727  * @apicid - apicid of vcpu to be kicked.
5728  */
5729 static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid)
5730 {
5731         struct kvm_lapic_irq lapic_irq;
5732
5733         lapic_irq.shorthand = 0;
5734         lapic_irq.dest_mode = 0;
5735         lapic_irq.dest_id = apicid;
5736
5737         lapic_irq.delivery_mode = APIC_DM_REMRD;
5738         kvm_irq_delivery_to_apic(kvm, 0, &lapic_irq, NULL);
5739 }
5740
5741 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
5742 {
5743         unsigned long nr, a0, a1, a2, a3, ret;
5744         int r = 1;
5745
5746         if (kvm_hv_hypercall_enabled(vcpu->kvm))
5747                 return kvm_hv_hypercall(vcpu);
5748
5749         nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
5750         a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
5751         a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
5752         a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
5753         a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
5754
5755         trace_kvm_hypercall(nr, a0, a1, a2, a3);
5756
5757         if (!is_long_mode(vcpu)) {
5758                 nr &= 0xFFFFFFFF;
5759                 a0 &= 0xFFFFFFFF;
5760                 a1 &= 0xFFFFFFFF;
5761                 a2 &= 0xFFFFFFFF;
5762                 a3 &= 0xFFFFFFFF;
5763         }
5764
5765         if (kvm_x86_ops->get_cpl(vcpu) != 0) {
5766                 ret = -KVM_EPERM;
5767                 goto out;
5768         }
5769
5770         switch (nr) {
5771         case KVM_HC_VAPIC_POLL_IRQ:
5772                 ret = 0;
5773                 break;
5774         case KVM_HC_KICK_CPU:
5775                 kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
5776                 ret = 0;
5777                 break;
5778         default:
5779                 ret = -KVM_ENOSYS;
5780                 break;
5781         }
5782 out:
5783         kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
5784         ++vcpu->stat.hypercalls;
5785         return r;
5786 }
5787 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
5788
5789 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
5790 {
5791         struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
5792         char instruction[3];
5793         unsigned long rip = kvm_rip_read(vcpu);
5794
5795         kvm_x86_ops->patch_hypercall(vcpu, instruction);
5796
5797         return emulator_write_emulated(ctxt, rip, instruction, 3, NULL);
5798 }
5799
5800 /*
5801  * Check if userspace requested an interrupt window, and that the
5802  * interrupt window is open.
5803  *
5804  * No need to exit to userspace if we already have an interrupt queued.
5805  */
5806 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
5807 {
5808         return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
5809                 vcpu->run->request_interrupt_window &&
5810                 kvm_arch_interrupt_allowed(vcpu));
5811 }
5812
5813 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
5814 {
5815         struct kvm_run *kvm_run = vcpu->run;
5816
5817         kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
5818         kvm_run->cr8 = kvm_get_cr8(vcpu);
5819         kvm_run->apic_base = kvm_get_apic_base(vcpu);
5820         if (irqchip_in_kernel(vcpu->kvm))
5821                 kvm_run->ready_for_interrupt_injection = 1;
5822         else
5823                 kvm_run->ready_for_interrupt_injection =
5824                         kvm_arch_interrupt_allowed(vcpu) &&
5825                         !kvm_cpu_has_interrupt(vcpu) &&
5826                         !kvm_event_needs_reinjection(vcpu);
5827 }
5828
5829 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
5830 {
5831         int max_irr, tpr;
5832
5833         if (!kvm_x86_ops->update_cr8_intercept)
5834                 return;
5835
5836         if (!vcpu->arch.apic)
5837                 return;
5838
5839         if (!vcpu->arch.apic->vapic_addr)
5840                 max_irr = kvm_lapic_find_highest_irr(vcpu);
5841         else
5842                 max_irr = -1;
5843
5844         if (max_irr != -1)
5845                 max_irr >>= 4;
5846
5847         tpr = kvm_lapic_get_cr8(vcpu);
5848
5849         kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
5850 }
5851
5852 static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
5853 {
5854         int r;
5855
5856         /* try to reinject previous events if any */
5857         if (vcpu->arch.exception.pending) {
5858                 trace_kvm_inj_exception(vcpu->arch.exception.nr,
5859                                         vcpu->arch.exception.has_error_code,
5860                                         vcpu->arch.exception.error_code);
5861                 kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
5862                                           vcpu->arch.exception.has_error_code,
5863                                           vcpu->arch.exception.error_code,
5864                                           vcpu->arch.exception.reinject);
5865                 return 0;
5866         }
5867
5868         if (vcpu->arch.nmi_injected) {
5869                 kvm_x86_ops->set_nmi(vcpu);
5870                 return 0;
5871         }
5872
5873         if (vcpu->arch.interrupt.pending) {
5874                 kvm_x86_ops->set_irq(vcpu);
5875                 return 0;
5876         }
5877
5878         if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) {
5879                 r = kvm_x86_ops->check_nested_events(vcpu, req_int_win);
5880                 if (r != 0)
5881                         return r;
5882         }
5883
5884         /* try to inject new event if pending */
5885         if (vcpu->arch.nmi_pending) {
5886                 if (kvm_x86_ops->nmi_allowed(vcpu)) {
5887                         --vcpu->arch.nmi_pending;
5888                         vcpu->arch.nmi_injected = true;
5889                         kvm_x86_ops->set_nmi(vcpu);
5890                 }
5891         } else if (kvm_cpu_has_injectable_intr(vcpu)) {
5892                 if (kvm_x86_ops->interrupt_allowed(vcpu)) {
5893                         kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
5894                                             false);
5895                         kvm_x86_ops->set_irq(vcpu);
5896                 }
5897         }
5898         return 0;
5899 }
5900
5901 static void process_nmi(struct kvm_vcpu *vcpu)
5902 {
5903         unsigned limit = 2;
5904
5905         /*
5906          * x86 is limited to one NMI running, and one NMI pending after it.
5907          * If an NMI is already in progress, limit further NMIs to just one.
5908          * Otherwise, allow two (and we'll inject the first one immediately).
5909          */
5910         if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected)
5911                 limit = 1;
5912
5913         vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
5914         vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
5915         kvm_make_request(KVM_REQ_EVENT, vcpu);
5916 }
5917
5918 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
5919 {
5920         u64 eoi_exit_bitmap[4];
5921         u32 tmr[8];
5922
5923         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
5924                 return;
5925
5926         memset(eoi_exit_bitmap, 0, 32);
5927         memset(tmr, 0, 32);
5928
5929         kvm_ioapic_scan_entry(vcpu, eoi_exit_bitmap, tmr);
5930         kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
5931         kvm_apic_update_tmr(vcpu, tmr);
5932 }
5933
5934 /*
5935  * Returns 1 to let __vcpu_run() continue the guest execution loop without
5936  * exiting to the userspace.  Otherwise, the value will be returned to the
5937  * userspace.
5938  */
5939 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
5940 {
5941         int r;
5942         bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
5943                 vcpu->run->request_interrupt_window;
5944         bool req_immediate_exit = false;
5945
5946         if (vcpu->requests) {
5947                 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
5948                         kvm_mmu_unload(vcpu);
5949                 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
5950                         __kvm_migrate_timers(vcpu);
5951                 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
5952                         kvm_gen_update_masterclock(vcpu->kvm);
5953                 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
5954                         kvm_gen_kvmclock_update(vcpu);
5955                 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
5956                         r = kvm_guest_time_update(vcpu);
5957                         if (unlikely(r))
5958                                 goto out;
5959                 }
5960                 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
5961                         kvm_mmu_sync_roots(vcpu);
5962                 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
5963                         kvm_x86_ops->tlb_flush(vcpu);
5964                 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
5965                         vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
5966                         r = 0;
5967                         goto out;
5968                 }
5969                 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
5970                         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
5971                         r = 0;
5972                         goto out;
5973                 }
5974                 if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) {
5975                         vcpu->fpu_active = 0;
5976                         kvm_x86_ops->fpu_deactivate(vcpu);
5977                 }
5978                 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
5979                         /* Page is swapped out. Do synthetic halt */
5980                         vcpu->arch.apf.halted = true;
5981                         r = 1;
5982                         goto out;
5983                 }
5984                 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
5985                         record_steal_time(vcpu);
5986                 if (kvm_check_request(KVM_REQ_NMI, vcpu))
5987                         process_nmi(vcpu);
5988                 if (kvm_check_request(KVM_REQ_PMU, vcpu))
5989                         kvm_handle_pmu_event(vcpu);
5990                 if (kvm_check_request(KVM_REQ_PMI, vcpu))
5991                         kvm_deliver_pmi(vcpu);
5992                 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
5993                         vcpu_scan_ioapic(vcpu);
5994         }
5995
5996         if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) {
5997                 kvm_apic_accept_events(vcpu);
5998                 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
5999                         r = 1;
6000                         goto out;
6001                 }
6002
6003                 if (inject_pending_event(vcpu, req_int_win) != 0)
6004                         req_immediate_exit = true;
6005                 /* enable NMI/IRQ window open exits if needed */
6006                 else if (vcpu->arch.nmi_pending)
6007                         kvm_x86_ops->enable_nmi_window(vcpu);
6008                 else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win)
6009                         kvm_x86_ops->enable_irq_window(vcpu);
6010
6011                 if (kvm_lapic_enabled(vcpu)) {
6012                         /*
6013                          * Update architecture specific hints for APIC
6014                          * virtual interrupt delivery.
6015                          */
6016                         if (kvm_x86_ops->hwapic_irr_update)
6017                                 kvm_x86_ops->hwapic_irr_update(vcpu,
6018                                         kvm_lapic_find_highest_irr(vcpu));
6019                         update_cr8_intercept(vcpu);
6020                         kvm_lapic_sync_to_vapic(vcpu);
6021                 }
6022         }
6023
6024         r = kvm_mmu_reload(vcpu);
6025         if (unlikely(r)) {
6026                 goto cancel_injection;
6027         }
6028
6029         preempt_disable();
6030
6031         kvm_x86_ops->prepare_guest_switch(vcpu);
6032         if (vcpu->fpu_active)
6033                 kvm_load_guest_fpu(vcpu);
6034         kvm_load_guest_xcr0(vcpu);
6035
6036         vcpu->mode = IN_GUEST_MODE;
6037
6038         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
6039
6040         /* We should set ->mode before check ->requests,
6041          * see the comment in make_all_cpus_request.
6042          */
6043         smp_mb__after_srcu_read_unlock();
6044
6045         local_irq_disable();
6046
6047         if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests
6048             || need_resched() || signal_pending(current)) {
6049                 vcpu->mode = OUTSIDE_GUEST_MODE;
6050                 smp_wmb();
6051                 local_irq_enable();
6052                 preempt_enable();
6053                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
6054                 r = 1;
6055                 goto cancel_injection;
6056         }
6057
6058         if (req_immediate_exit)
6059                 smp_send_reschedule(vcpu->cpu);
6060
6061         kvm_guest_enter();
6062
6063         if (unlikely(vcpu->arch.switch_db_regs)) {
6064                 set_debugreg(0, 7);
6065                 set_debugreg(vcpu->arch.eff_db[0], 0);
6066                 set_debugreg(vcpu->arch.eff_db[1], 1);
6067                 set_debugreg(vcpu->arch.eff_db[2], 2);
6068                 set_debugreg(vcpu->arch.eff_db[3], 3);
6069                 set_debugreg(vcpu->arch.dr6, 6);
6070         }
6071
6072         trace_kvm_entry(vcpu->vcpu_id);
6073         kvm_x86_ops->run(vcpu);
6074
6075         /*
6076          * Do this here before restoring debug registers on the host.  And
6077          * since we do this before handling the vmexit, a DR access vmexit
6078          * can (a) read the correct value of the debug registers, (b) set
6079          * KVM_DEBUGREG_WONT_EXIT again.
6080          */
6081         if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
6082                 int i;
6083
6084                 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
6085                 kvm_x86_ops->sync_dirty_debug_regs(vcpu);
6086                 for (i = 0; i < KVM_NR_DB_REGS; i++)
6087                         vcpu->arch.eff_db[i] = vcpu->arch.db[i];
6088         }
6089
6090         /*
6091          * If the guest has used debug registers, at least dr7
6092          * will be disabled while returning to the host.
6093          * If we don't have active breakpoints in the host, we don't
6094          * care about the messed up debug address registers. But if
6095          * we have some of them active, restore the old state.
6096          */
6097         if (hw_breakpoint_active())
6098                 hw_breakpoint_restore();
6099
6100         vcpu->arch.last_guest_tsc = kvm_x86_ops->read_l1_tsc(vcpu,
6101                                                            native_read_tsc());
6102
6103         vcpu->mode = OUTSIDE_GUEST_MODE;
6104         smp_wmb();
6105
6106         /* Interrupt is enabled by handle_external_intr() */
6107         kvm_x86_ops->handle_external_intr(vcpu);
6108
6109         ++vcpu->stat.exits;
6110
6111         /*
6112          * We must have an instruction between local_irq_enable() and
6113          * kvm_guest_exit(), so the timer interrupt isn't delayed by
6114          * the interrupt shadow.  The stat.exits increment will do nicely.
6115          * But we need to prevent reordering, hence this barrier():
6116          */
6117         barrier();
6118
6119         kvm_guest_exit();
6120
6121         preempt_enable();
6122
6123         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
6124
6125         /*
6126          * Profile KVM exit RIPs:
6127          */
6128         if (unlikely(prof_on == KVM_PROFILING)) {
6129                 unsigned long rip = kvm_rip_read(vcpu);
6130                 profile_hit(KVM_PROFILING, (void *)rip);
6131         }
6132
6133         if (unlikely(vcpu->arch.tsc_always_catchup))
6134                 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
6135
6136         if (vcpu->arch.apic_attention)
6137                 kvm_lapic_sync_from_vapic(vcpu);
6138
6139         r = kvm_x86_ops->handle_exit(vcpu);
6140         return r;
6141
6142 cancel_injection:
6143         kvm_x86_ops->cancel_injection(vcpu);
6144         if (unlikely(vcpu->arch.apic_attention))
6145                 kvm_lapic_sync_from_vapic(vcpu);
6146 out:
6147         return r;
6148 }
6149
6150
6151 static int __vcpu_run(struct kvm_vcpu *vcpu)
6152 {
6153         int r;
6154         struct kvm *kvm = vcpu->kvm;
6155
6156         vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6157
6158         r = 1;
6159         while (r > 0) {
6160                 if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
6161                     !vcpu->arch.apf.halted)
6162                         r = vcpu_enter_guest(vcpu);
6163                 else {
6164                         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6165                         kvm_vcpu_block(vcpu);
6166                         vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6167                         if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
6168                                 kvm_apic_accept_events(vcpu);
6169                                 switch(vcpu->arch.mp_state) {
6170                                 case KVM_MP_STATE_HALTED:
6171                                         vcpu->arch.pv.pv_unhalted = false;
6172                                         vcpu->arch.mp_state =
6173                                                 KVM_MP_STATE_RUNNABLE;
6174                                 case KVM_MP_STATE_RUNNABLE:
6175                                         vcpu->arch.apf.halted = false;
6176                                         break;
6177                                 case KVM_MP_STATE_INIT_RECEIVED:
6178                                         break;
6179                                 default:
6180                                         r = -EINTR;
6181                                         break;
6182                                 }
6183                         }
6184                 }
6185
6186                 if (r <= 0)
6187                         break;
6188
6189                 clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
6190                 if (kvm_cpu_has_pending_timer(vcpu))
6191                         kvm_inject_pending_timer_irqs(vcpu);
6192
6193                 if (dm_request_for_irq_injection(vcpu)) {
6194                         r = -EINTR;
6195                         vcpu->run->exit_reason = KVM_EXIT_INTR;
6196                         ++vcpu->stat.request_irq_exits;
6197                 }
6198
6199                 kvm_check_async_pf_completion(vcpu);
6200
6201                 if (signal_pending(current)) {
6202                         r = -EINTR;
6203                         vcpu->run->exit_reason = KVM_EXIT_INTR;
6204                         ++vcpu->stat.signal_exits;
6205                 }
6206                 if (need_resched()) {
6207                         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6208                         cond_resched();
6209                         vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
6210                 }
6211         }
6212
6213         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
6214
6215         return r;
6216 }
6217
6218 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
6219 {
6220         int r;
6221         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
6222         r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
6223         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
6224         if (r != EMULATE_DONE)
6225                 return 0;
6226         return 1;
6227 }
6228
6229 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
6230 {
6231         BUG_ON(!vcpu->arch.pio.count);
6232
6233         return complete_emulated_io(vcpu);
6234 }
6235
6236 /*
6237  * Implements the following, as a state machine:
6238  *
6239  * read:
6240  *   for each fragment
6241  *     for each mmio piece in the fragment
6242  *       write gpa, len
6243  *       exit
6244  *       copy data
6245  *   execute insn
6246  *
6247  * write:
6248  *   for each fragment
6249  *     for each mmio piece in the fragment
6250  *       write gpa, len
6251  *       copy data
6252  *       exit
6253  */
6254 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
6255 {
6256         struct kvm_run *run = vcpu->run;
6257         struct kvm_mmio_fragment *frag;
6258         unsigned len;
6259
6260         BUG_ON(!vcpu->mmio_needed);
6261
6262         /* Complete previous fragment */
6263         frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
6264         len = min(8u, frag->len);
6265         if (!vcpu->mmio_is_write)
6266                 memcpy(frag->data, run->mmio.data, len);
6267
6268         if (frag->len <= 8) {
6269                 /* Switch to the next fragment. */
6270                 frag++;
6271                 vcpu->mmio_cur_fragment++;
6272         } else {
6273                 /* Go forward to the next mmio piece. */
6274                 frag->data += len;
6275                 frag->gpa += len;
6276                 frag->len -= len;
6277         }
6278
6279         if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
6280                 vcpu->mmio_needed = 0;
6281
6282                 /* FIXME: return into emulator if single-stepping.  */
6283                 if (vcpu->mmio_is_write)
6284                         return 1;
6285                 vcpu->mmio_read_completed = 1;
6286                 return complete_emulated_io(vcpu);
6287         }
6288
6289         run->exit_reason = KVM_EXIT_MMIO;
6290         run->mmio.phys_addr = frag->gpa;
6291         if (vcpu->mmio_is_write)
6292                 memcpy(run->mmio.data, frag->data, min(8u, frag->len));
6293         run->mmio.len = min(8u, frag->len);
6294         run->mmio.is_write = vcpu->mmio_is_write;
6295         vcpu->arch.complete_userspace_io = complete_emulated_mmio;
6296         return 0;
6297 }
6298
6299
6300 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6301 {
6302         int r;
6303         sigset_t sigsaved;
6304
6305         if (!tsk_used_math(current) && init_fpu(current))
6306                 return -ENOMEM;
6307
6308         if (vcpu->sigset_active)
6309                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
6310
6311         if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
6312                 kvm_vcpu_block(vcpu);
6313                 kvm_apic_accept_events(vcpu);
6314                 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
6315                 r = -EAGAIN;
6316                 goto out;
6317         }
6318
6319         /* re-sync apic's tpr */
6320         if (!irqchip_in_kernel(vcpu->kvm)) {
6321                 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
6322                         r = -EINVAL;
6323                         goto out;
6324                 }
6325         }
6326
6327         if (unlikely(vcpu->arch.complete_userspace_io)) {
6328                 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
6329                 vcpu->arch.complete_userspace_io = NULL;
6330                 r = cui(vcpu);
6331                 if (r <= 0)
6332                         goto out;
6333         } else
6334                 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
6335
6336         r = __vcpu_run(vcpu);
6337
6338 out:
6339         post_kvm_run_save(vcpu);
6340         if (vcpu->sigset_active)
6341                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
6342
6343         return r;
6344 }
6345
6346 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
6347 {
6348         if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
6349                 /*
6350                  * We are here if userspace calls get_regs() in the middle of
6351                  * instruction emulation. Registers state needs to be copied
6352                  * back from emulation context to vcpu. Userspace shouldn't do
6353                  * that usually, but some bad designed PV devices (vmware
6354                  * backdoor interface) need this to work
6355                  */
6356                 emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
6357                 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6358         }
6359         regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
6360         regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
6361         regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
6362         regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
6363         regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
6364         regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
6365         regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
6366         regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
6367 #ifdef CONFIG_X86_64
6368         regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
6369         regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
6370         regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
6371         regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
6372         regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
6373         regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
6374         regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
6375         regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
6376 #endif
6377
6378         regs->rip = kvm_rip_read(vcpu);
6379         regs->rflags = kvm_get_rflags(vcpu);
6380
6381         return 0;
6382 }
6383
6384 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
6385 {
6386         vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
6387         vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
6388
6389         kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
6390         kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
6391         kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
6392         kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
6393         kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
6394         kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
6395         kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
6396         kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
6397 #ifdef CONFIG_X86_64
6398         kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
6399         kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
6400         kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
6401         kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
6402         kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
6403         kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
6404         kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
6405         kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
6406 #endif
6407
6408         kvm_rip_write(vcpu, regs->rip);
6409         kvm_set_rflags(vcpu, regs->rflags);
6410
6411         vcpu->arch.exception.pending = false;
6412
6413         kvm_make_request(KVM_REQ_EVENT, vcpu);
6414
6415         return 0;
6416 }
6417
6418 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
6419 {
6420         struct kvm_segment cs;
6421
6422         kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
6423         *db = cs.db;
6424         *l = cs.l;
6425 }
6426 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
6427
6428 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
6429                                   struct kvm_sregs *sregs)
6430 {
6431         struct desc_ptr dt;
6432
6433         kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
6434         kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
6435         kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
6436         kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
6437         kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
6438         kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6439
6440         kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
6441         kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6442
6443         kvm_x86_ops->get_idt(vcpu, &dt);
6444         sregs->idt.limit = dt.size;
6445         sregs->idt.base = dt.address;
6446         kvm_x86_ops->get_gdt(vcpu, &dt);
6447         sregs->gdt.limit = dt.size;
6448         sregs->gdt.base = dt.address;
6449
6450         sregs->cr0 = kvm_read_cr0(vcpu);
6451         sregs->cr2 = vcpu->arch.cr2;
6452         sregs->cr3 = kvm_read_cr3(vcpu);
6453         sregs->cr4 = kvm_read_cr4(vcpu);
6454         sregs->cr8 = kvm_get_cr8(vcpu);
6455         sregs->efer = vcpu->arch.efer;
6456         sregs->apic_base = kvm_get_apic_base(vcpu);
6457
6458         memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
6459
6460         if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
6461                 set_bit(vcpu->arch.interrupt.nr,
6462                         (unsigned long *)sregs->interrupt_bitmap);
6463
6464         return 0;
6465 }
6466
6467 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
6468                                     struct kvm_mp_state *mp_state)
6469 {
6470         kvm_apic_accept_events(vcpu);
6471         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
6472                                         vcpu->arch.pv.pv_unhalted)
6473                 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
6474         else
6475                 mp_state->mp_state = vcpu->arch.mp_state;
6476
6477         return 0;
6478 }
6479
6480 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
6481                                     struct kvm_mp_state *mp_state)
6482 {
6483         if (!kvm_vcpu_has_lapic(vcpu) &&
6484             mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
6485                 return -EINVAL;
6486
6487         if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
6488                 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
6489                 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
6490         } else
6491                 vcpu->arch.mp_state = mp_state->mp_state;
6492         kvm_make_request(KVM_REQ_EVENT, vcpu);
6493         return 0;
6494 }
6495
6496 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
6497                     int reason, bool has_error_code, u32 error_code)
6498 {
6499         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
6500         int ret;
6501
6502         init_emulate_ctxt(vcpu);
6503
6504         ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
6505                                    has_error_code, error_code);
6506
6507         if (ret)
6508                 return EMULATE_FAIL;
6509
6510         kvm_rip_write(vcpu, ctxt->eip);
6511         kvm_set_rflags(vcpu, ctxt->eflags);
6512         kvm_make_request(KVM_REQ_EVENT, vcpu);
6513         return EMULATE_DONE;
6514 }
6515 EXPORT_SYMBOL_GPL(kvm_task_switch);
6516
6517 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
6518                                   struct kvm_sregs *sregs)
6519 {
6520         struct msr_data apic_base_msr;
6521         int mmu_reset_needed = 0;
6522         int pending_vec, max_bits, idx;
6523         struct desc_ptr dt;
6524
6525         if (!guest_cpuid_has_xsave(vcpu) && (sregs->cr4 & X86_CR4_OSXSAVE))
6526                 return -EINVAL;
6527
6528         dt.size = sregs->idt.limit;
6529         dt.address = sregs->idt.base;
6530         kvm_x86_ops->set_idt(vcpu, &dt);
6531         dt.size = sregs->gdt.limit;
6532         dt.address = sregs->gdt.base;
6533         kvm_x86_ops->set_gdt(vcpu, &dt);
6534
6535         vcpu->arch.cr2 = sregs->cr2;
6536         mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
6537         vcpu->arch.cr3 = sregs->cr3;
6538         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
6539
6540         kvm_set_cr8(vcpu, sregs->cr8);
6541
6542         mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
6543         kvm_x86_ops->set_efer(vcpu, sregs->efer);
6544         apic_base_msr.data = sregs->apic_base;
6545         apic_base_msr.host_initiated = true;
6546         kvm_set_apic_base(vcpu, &apic_base_msr);
6547
6548         mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
6549         kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
6550         vcpu->arch.cr0 = sregs->cr0;
6551
6552         mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
6553         kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
6554         if (sregs->cr4 & X86_CR4_OSXSAVE)
6555                 kvm_update_cpuid(vcpu);
6556
6557         idx = srcu_read_lock(&vcpu->kvm->srcu);
6558         if (!is_long_mode(vcpu) && is_pae(vcpu)) {
6559                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
6560                 mmu_reset_needed = 1;
6561         }
6562         srcu_read_unlock(&vcpu->kvm->srcu, idx);
6563
6564         if (mmu_reset_needed)
6565                 kvm_mmu_reset_context(vcpu);
6566
6567         max_bits = KVM_NR_INTERRUPTS;
6568         pending_vec = find_first_bit(
6569                 (const unsigned long *)sregs->interrupt_bitmap, max_bits);
6570         if (pending_vec < max_bits) {
6571                 kvm_queue_interrupt(vcpu, pending_vec, false);
6572                 pr_debug("Set back pending irq %d\n", pending_vec);
6573         }
6574
6575         kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
6576         kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
6577         kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
6578         kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
6579         kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
6580         kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
6581
6582         kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
6583         kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
6584
6585         update_cr8_intercept(vcpu);
6586
6587         /* Older userspace won't unhalt the vcpu on reset. */
6588         if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
6589             sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
6590             !is_protmode(vcpu))
6591                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
6592
6593         kvm_make_request(KVM_REQ_EVENT, vcpu);
6594
6595         return 0;
6596 }
6597
6598 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
6599                                         struct kvm_guest_debug *dbg)
6600 {
6601         unsigned long rflags;
6602         int i, r;
6603
6604         if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
6605                 r = -EBUSY;
6606                 if (vcpu->arch.exception.pending)
6607                         goto out;
6608                 if (dbg->control & KVM_GUESTDBG_INJECT_DB)
6609                         kvm_queue_exception(vcpu, DB_VECTOR);
6610                 else
6611                         kvm_queue_exception(vcpu, BP_VECTOR);
6612         }
6613
6614         /*
6615          * Read rflags as long as potentially injected trace flags are still
6616          * filtered out.
6617          */
6618         rflags = kvm_get_rflags(vcpu);
6619
6620         vcpu->guest_debug = dbg->control;
6621         if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
6622                 vcpu->guest_debug = 0;
6623
6624         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
6625                 for (i = 0; i < KVM_NR_DB_REGS; ++i)
6626                         vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
6627                 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
6628         } else {
6629                 for (i = 0; i < KVM_NR_DB_REGS; i++)
6630                         vcpu->arch.eff_db[i] = vcpu->arch.db[i];
6631         }
6632         kvm_update_dr7(vcpu);
6633
6634         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
6635                 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
6636                         get_segment_base(vcpu, VCPU_SREG_CS);
6637
6638         /*
6639          * Trigger an rflags update that will inject or remove the trace
6640          * flags.
6641          */
6642         kvm_set_rflags(vcpu, rflags);
6643
6644         kvm_x86_ops->update_db_bp_intercept(vcpu);
6645
6646         r = 0;
6647
6648 out:
6649
6650         return r;
6651 }
6652
6653 /*
6654  * Translate a guest virtual address to a guest physical address.
6655  */
6656 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
6657                                     struct kvm_translation *tr)
6658 {
6659         unsigned long vaddr = tr->linear_address;
6660         gpa_t gpa;
6661         int idx;
6662
6663         idx = srcu_read_lock(&vcpu->kvm->srcu);
6664         gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
6665         srcu_read_unlock(&vcpu->kvm->srcu, idx);
6666         tr->physical_address = gpa;
6667         tr->valid = gpa != UNMAPPED_GVA;
6668         tr->writeable = 1;
6669         tr->usermode = 0;
6670
6671         return 0;
6672 }
6673
6674 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
6675 {
6676         struct i387_fxsave_struct *fxsave =
6677                         &vcpu->arch.guest_fpu.state->fxsave;
6678
6679         memcpy(fpu->fpr, fxsave->st_space, 128);
6680         fpu->fcw = fxsave->cwd;
6681         fpu->fsw = fxsave->swd;
6682         fpu->ftwx = fxsave->twd;
6683         fpu->last_opcode = fxsave->fop;
6684         fpu->last_ip = fxsave->rip;
6685         fpu->last_dp = fxsave->rdp;
6686         memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
6687
6688         return 0;
6689 }
6690
6691 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
6692 {
6693         struct i387_fxsave_struct *fxsave =
6694                         &vcpu->arch.guest_fpu.state->fxsave;
6695
6696         memcpy(fxsave->st_space, fpu->fpr, 128);
6697         fxsave->cwd = fpu->fcw;
6698         fxsave->swd = fpu->fsw;
6699         fxsave->twd = fpu->ftwx;
6700         fxsave->fop = fpu->last_opcode;
6701         fxsave->rip = fpu->last_ip;
6702         fxsave->rdp = fpu->last_dp;
6703         memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
6704
6705         return 0;
6706 }
6707
6708 int fx_init(struct kvm_vcpu *vcpu)
6709 {
6710         int err;
6711
6712         err = fpu_alloc(&vcpu->arch.guest_fpu);
6713         if (err)
6714                 return err;
6715
6716         fpu_finit(&vcpu->arch.guest_fpu);
6717
6718         /*
6719          * Ensure guest xcr0 is valid for loading
6720          */
6721         vcpu->arch.xcr0 = XSTATE_FP;
6722
6723         vcpu->arch.cr0 |= X86_CR0_ET;
6724
6725         return 0;
6726 }
6727 EXPORT_SYMBOL_GPL(fx_init);
6728
6729 static void fx_free(struct kvm_vcpu *vcpu)
6730 {
6731         fpu_free(&vcpu->arch.guest_fpu);
6732 }
6733
6734 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
6735 {
6736         if (vcpu->guest_fpu_loaded)
6737                 return;
6738
6739         /*
6740          * Restore all possible states in the guest,
6741          * and assume host would use all available bits.
6742          * Guest xcr0 would be loaded later.
6743          */
6744         kvm_put_guest_xcr0(vcpu);
6745         vcpu->guest_fpu_loaded = 1;
6746         __kernel_fpu_begin();
6747         fpu_restore_checking(&vcpu->arch.guest_fpu);
6748         trace_kvm_fpu(1);
6749 }
6750
6751 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
6752 {
6753         kvm_put_guest_xcr0(vcpu);
6754
6755         if (!vcpu->guest_fpu_loaded)
6756                 return;
6757
6758         vcpu->guest_fpu_loaded = 0;
6759         fpu_save_init(&vcpu->arch.guest_fpu);
6760         __kernel_fpu_end();
6761         ++vcpu->stat.fpu_reload;
6762         kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
6763         trace_kvm_fpu(0);
6764 }
6765
6766 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
6767 {
6768         kvmclock_reset(vcpu);
6769
6770         free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
6771         fx_free(vcpu);
6772         kvm_x86_ops->vcpu_free(vcpu);
6773 }
6774
6775 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
6776                                                 unsigned int id)
6777 {
6778         if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
6779                 printk_once(KERN_WARNING
6780                 "kvm: SMP vm created on host with unstable TSC; "
6781                 "guest TSC will not be reliable\n");
6782         return kvm_x86_ops->vcpu_create(kvm, id);
6783 }
6784
6785 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
6786 {
6787         int r;
6788
6789         vcpu->arch.mtrr_state.have_fixed = 1;
6790         r = vcpu_load(vcpu);
6791         if (r)
6792                 return r;
6793         kvm_vcpu_reset(vcpu);
6794         kvm_mmu_setup(vcpu);
6795         vcpu_put(vcpu);
6796
6797         return r;
6798 }
6799
6800 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
6801 {
6802         int r;
6803         struct msr_data msr;
6804         struct kvm *kvm = vcpu->kvm;
6805
6806         r = vcpu_load(vcpu);
6807         if (r)
6808                 return r;
6809         msr.data = 0x0;
6810         msr.index = MSR_IA32_TSC;
6811         msr.host_initiated = true;
6812         kvm_write_tsc(vcpu, &msr);
6813         vcpu_put(vcpu);
6814
6815         schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
6816                                         KVMCLOCK_SYNC_PERIOD);
6817
6818         return r;
6819 }
6820
6821 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
6822 {
6823         int r;
6824         vcpu->arch.apf.msr_val = 0;
6825
6826         r = vcpu_load(vcpu);
6827         BUG_ON(r);
6828         kvm_mmu_unload(vcpu);
6829         vcpu_put(vcpu);
6830
6831         fx_free(vcpu);
6832         kvm_x86_ops->vcpu_free(vcpu);
6833 }
6834
6835 void kvm_vcpu_reset(struct kvm_vcpu *vcpu)
6836 {
6837         atomic_set(&vcpu->arch.nmi_queued, 0);
6838         vcpu->arch.nmi_pending = 0;
6839         vcpu->arch.nmi_injected = false;
6840
6841         memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
6842         vcpu->arch.dr6 = DR6_FIXED_1;
6843         kvm_update_dr6(vcpu);
6844         vcpu->arch.dr7 = DR7_FIXED_1;
6845         kvm_update_dr7(vcpu);
6846
6847         kvm_make_request(KVM_REQ_EVENT, vcpu);
6848         vcpu->arch.apf.msr_val = 0;
6849         vcpu->arch.st.msr_val = 0;
6850
6851         kvmclock_reset(vcpu);
6852
6853         kvm_clear_async_pf_completion_queue(vcpu);
6854         kvm_async_pf_hash_reset(vcpu);
6855         vcpu->arch.apf.halted = false;
6856
6857         kvm_pmu_reset(vcpu);
6858
6859         memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
6860         vcpu->arch.regs_avail = ~0;
6861         vcpu->arch.regs_dirty = ~0;
6862
6863         kvm_x86_ops->vcpu_reset(vcpu);
6864 }
6865
6866 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, unsigned int vector)
6867 {
6868         struct kvm_segment cs;
6869
6870         kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
6871         cs.selector = vector << 8;
6872         cs.base = vector << 12;
6873         kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
6874         kvm_rip_write(vcpu, 0);
6875 }
6876
6877 int kvm_arch_hardware_enable(void *garbage)
6878 {
6879         struct kvm *kvm;
6880         struct kvm_vcpu *vcpu;
6881         int i;
6882         int ret;
6883         u64 local_tsc;
6884         u64 max_tsc = 0;
6885         bool stable, backwards_tsc = false;
6886
6887         kvm_shared_msr_cpu_online();
6888         ret = kvm_x86_ops->hardware_enable(garbage);
6889         if (ret != 0)
6890                 return ret;
6891
6892         local_tsc = native_read_tsc();
6893         stable = !check_tsc_unstable();
6894         list_for_each_entry(kvm, &vm_list, vm_list) {
6895                 kvm_for_each_vcpu(i, vcpu, kvm) {
6896                         if (!stable && vcpu->cpu == smp_processor_id())
6897                                 set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
6898                         if (stable && vcpu->arch.last_host_tsc > local_tsc) {
6899                                 backwards_tsc = true;
6900                                 if (vcpu->arch.last_host_tsc > max_tsc)
6901                                         max_tsc = vcpu->arch.last_host_tsc;
6902                         }
6903                 }
6904         }
6905
6906         /*
6907          * Sometimes, even reliable TSCs go backwards.  This happens on
6908          * platforms that reset TSC during suspend or hibernate actions, but
6909          * maintain synchronization.  We must compensate.  Fortunately, we can
6910          * detect that condition here, which happens early in CPU bringup,
6911          * before any KVM threads can be running.  Unfortunately, we can't
6912          * bring the TSCs fully up to date with real time, as we aren't yet far
6913          * enough into CPU bringup that we know how much real time has actually
6914          * elapsed; our helper function, get_kernel_ns() will be using boot
6915          * variables that haven't been updated yet.
6916          *
6917          * So we simply find the maximum observed TSC above, then record the
6918          * adjustment to TSC in each VCPU.  When the VCPU later gets loaded,
6919          * the adjustment will be applied.  Note that we accumulate
6920          * adjustments, in case multiple suspend cycles happen before some VCPU
6921          * gets a chance to run again.  In the event that no KVM threads get a
6922          * chance to run, we will miss the entire elapsed period, as we'll have
6923          * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
6924          * loose cycle time.  This isn't too big a deal, since the loss will be
6925          * uniform across all VCPUs (not to mention the scenario is extremely
6926          * unlikely). It is possible that a second hibernate recovery happens
6927          * much faster than a first, causing the observed TSC here to be
6928          * smaller; this would require additional padding adjustment, which is
6929          * why we set last_host_tsc to the local tsc observed here.
6930          *
6931          * N.B. - this code below runs only on platforms with reliable TSC,
6932          * as that is the only way backwards_tsc is set above.  Also note
6933          * that this runs for ALL vcpus, which is not a bug; all VCPUs should
6934          * have the same delta_cyc adjustment applied if backwards_tsc
6935          * is detected.  Note further, this adjustment is only done once,
6936          * as we reset last_host_tsc on all VCPUs to stop this from being
6937          * called multiple times (one for each physical CPU bringup).
6938          *
6939          * Platforms with unreliable TSCs don't have to deal with this, they
6940          * will be compensated by the logic in vcpu_load, which sets the TSC to
6941          * catchup mode.  This will catchup all VCPUs to real time, but cannot
6942          * guarantee that they stay in perfect synchronization.
6943          */
6944         if (backwards_tsc) {
6945                 u64 delta_cyc = max_tsc - local_tsc;
6946                 list_for_each_entry(kvm, &vm_list, vm_list) {
6947                         kvm_for_each_vcpu(i, vcpu, kvm) {
6948                                 vcpu->arch.tsc_offset_adjustment += delta_cyc;
6949                                 vcpu->arch.last_host_tsc = local_tsc;
6950                                 set_bit(KVM_REQ_MASTERCLOCK_UPDATE,
6951                                         &vcpu->requests);
6952                         }
6953
6954                         /*
6955                          * We have to disable TSC offset matching.. if you were
6956                          * booting a VM while issuing an S4 host suspend....
6957                          * you may have some problem.  Solving this issue is
6958                          * left as an exercise to the reader.
6959                          */
6960                         kvm->arch.last_tsc_nsec = 0;
6961                         kvm->arch.last_tsc_write = 0;
6962                 }
6963
6964         }
6965         return 0;
6966 }
6967
6968 void kvm_arch_hardware_disable(void *garbage)
6969 {
6970         kvm_x86_ops->hardware_disable(garbage);
6971         drop_user_return_notifiers(garbage);
6972 }
6973
6974 int kvm_arch_hardware_setup(void)
6975 {
6976         return kvm_x86_ops->hardware_setup();
6977 }
6978
6979 void kvm_arch_hardware_unsetup(void)
6980 {
6981         kvm_x86_ops->hardware_unsetup();
6982 }
6983
6984 void kvm_arch_check_processor_compat(void *rtn)
6985 {
6986         kvm_x86_ops->check_processor_compatibility(rtn);
6987 }
6988
6989 bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
6990 {
6991         return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
6992 }
6993
6994 struct static_key kvm_no_apic_vcpu __read_mostly;
6995
6996 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
6997 {
6998         struct page *page;
6999         struct kvm *kvm;
7000         int r;
7001
7002         BUG_ON(vcpu->kvm == NULL);
7003         kvm = vcpu->kvm;
7004
7005         vcpu->arch.pv.pv_unhalted = false;
7006         vcpu->arch.emulate_ctxt.ops = &emulate_ops;
7007         if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
7008                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7009         else
7010                 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
7011
7012         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
7013         if (!page) {
7014                 r = -ENOMEM;
7015                 goto fail;
7016         }
7017         vcpu->arch.pio_data = page_address(page);
7018
7019         kvm_set_tsc_khz(vcpu, max_tsc_khz);
7020
7021         r = kvm_mmu_create(vcpu);
7022         if (r < 0)
7023                 goto fail_free_pio_data;
7024
7025         if (irqchip_in_kernel(kvm)) {
7026                 r = kvm_create_lapic(vcpu);
7027                 if (r < 0)
7028                         goto fail_mmu_destroy;
7029         } else
7030                 static_key_slow_inc(&kvm_no_apic_vcpu);
7031
7032         vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
7033                                        GFP_KERNEL);
7034         if (!vcpu->arch.mce_banks) {
7035                 r = -ENOMEM;
7036                 goto fail_free_lapic;
7037         }
7038         vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
7039
7040         if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) {
7041                 r = -ENOMEM;
7042                 goto fail_free_mce_banks;
7043         }
7044
7045         r = fx_init(vcpu);
7046         if (r)
7047                 goto fail_free_wbinvd_dirty_mask;
7048
7049         vcpu->arch.ia32_tsc_adjust_msr = 0x0;
7050         vcpu->arch.pv_time_enabled = false;
7051
7052         vcpu->arch.guest_supported_xcr0 = 0;
7053         vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
7054
7055         kvm_async_pf_hash_reset(vcpu);
7056         kvm_pmu_init(vcpu);
7057
7058         return 0;
7059 fail_free_wbinvd_dirty_mask:
7060         free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
7061 fail_free_mce_banks:
7062         kfree(vcpu->arch.mce_banks);
7063 fail_free_lapic:
7064         kvm_free_lapic(vcpu);
7065 fail_mmu_destroy:
7066         kvm_mmu_destroy(vcpu);
7067 fail_free_pio_data:
7068         free_page((unsigned long)vcpu->arch.pio_data);
7069 fail:
7070         return r;
7071 }
7072
7073 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
7074 {
7075         int idx;
7076
7077         kvm_pmu_destroy(vcpu);
7078         kfree(vcpu->arch.mce_banks);
7079         kvm_free_lapic(vcpu);
7080         idx = srcu_read_lock(&vcpu->kvm->srcu);
7081         kvm_mmu_destroy(vcpu);
7082         srcu_read_unlock(&vcpu->kvm->srcu, idx);
7083         free_page((unsigned long)vcpu->arch.pio_data);
7084         if (!irqchip_in_kernel(vcpu->kvm))
7085                 static_key_slow_dec(&kvm_no_apic_vcpu);
7086 }
7087
7088 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
7089 {
7090         if (type)
7091                 return -EINVAL;
7092
7093         INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
7094         INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
7095         INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
7096         atomic_set(&kvm->arch.noncoherent_dma_count, 0);
7097
7098         /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
7099         set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
7100         /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
7101         set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
7102                 &kvm->arch.irq_sources_bitmap);
7103
7104         raw_spin_lock_init(&kvm->arch.tsc_write_lock);
7105         mutex_init(&kvm->arch.apic_map_lock);
7106         spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock);
7107
7108         pvclock_update_vm_gtod_copy(kvm);
7109
7110         INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
7111         INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
7112
7113         return 0;
7114 }
7115
7116 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
7117 {
7118         int r;
7119         r = vcpu_load(vcpu);
7120         BUG_ON(r);
7121         kvm_mmu_unload(vcpu);
7122         vcpu_put(vcpu);
7123 }
7124
7125 static void kvm_free_vcpus(struct kvm *kvm)
7126 {
7127         unsigned int i;
7128         struct kvm_vcpu *vcpu;
7129
7130         /*
7131          * Unpin any mmu pages first.
7132          */
7133         kvm_for_each_vcpu(i, vcpu, kvm) {
7134                 kvm_clear_async_pf_completion_queue(vcpu);
7135                 kvm_unload_vcpu_mmu(vcpu);
7136         }
7137         kvm_for_each_vcpu(i, vcpu, kvm)
7138                 kvm_arch_vcpu_free(vcpu);
7139
7140         mutex_lock(&kvm->lock);
7141         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
7142                 kvm->vcpus[i] = NULL;
7143
7144         atomic_set(&kvm->online_vcpus, 0);
7145         mutex_unlock(&kvm->lock);
7146 }
7147
7148 void kvm_arch_sync_events(struct kvm *kvm)
7149 {
7150         cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
7151         cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
7152         kvm_free_all_assigned_devices(kvm);
7153         kvm_free_pit(kvm);
7154 }
7155
7156 void kvm_arch_destroy_vm(struct kvm *kvm)
7157 {
7158         if (current->mm == kvm->mm) {
7159                 /*
7160                  * Free memory regions allocated on behalf of userspace,
7161                  * unless the the memory map has changed due to process exit
7162                  * or fd copying.
7163                  */
7164                 struct kvm_userspace_memory_region mem;
7165                 memset(&mem, 0, sizeof(mem));
7166                 mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
7167                 kvm_set_memory_region(kvm, &mem);
7168
7169                 mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
7170                 kvm_set_memory_region(kvm, &mem);
7171
7172                 mem.slot = TSS_PRIVATE_MEMSLOT;
7173                 kvm_set_memory_region(kvm, &mem);
7174         }
7175         kvm_iommu_unmap_guest(kvm);
7176         kfree(kvm->arch.vpic);
7177         kfree(kvm->arch.vioapic);
7178         kvm_free_vcpus(kvm);
7179         if (kvm->arch.apic_access_page)
7180                 put_page(kvm->arch.apic_access_page);
7181         if (kvm->arch.ept_identity_pagetable)
7182                 put_page(kvm->arch.ept_identity_pagetable);
7183         kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
7184 }
7185
7186 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
7187                            struct kvm_memory_slot *dont)
7188 {
7189         int i;
7190
7191         for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7192                 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
7193                         kvm_kvfree(free->arch.rmap[i]);
7194                         free->arch.rmap[i] = NULL;
7195                 }
7196                 if (i == 0)
7197                         continue;
7198
7199                 if (!dont || free->arch.lpage_info[i - 1] !=
7200                              dont->arch.lpage_info[i - 1]) {
7201                         kvm_kvfree(free->arch.lpage_info[i - 1]);
7202                         free->arch.lpage_info[i - 1] = NULL;
7203                 }
7204         }
7205 }
7206
7207 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
7208                             unsigned long npages)
7209 {
7210         int i;
7211
7212         for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7213                 unsigned long ugfn;
7214                 int lpages;
7215                 int level = i + 1;
7216
7217                 lpages = gfn_to_index(slot->base_gfn + npages - 1,
7218                                       slot->base_gfn, level) + 1;
7219
7220                 slot->arch.rmap[i] =
7221                         kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
7222                 if (!slot->arch.rmap[i])
7223                         goto out_free;
7224                 if (i == 0)
7225                         continue;
7226
7227                 slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
7228                                         sizeof(*slot->arch.lpage_info[i - 1]));
7229                 if (!slot->arch.lpage_info[i - 1])
7230                         goto out_free;
7231
7232                 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
7233                         slot->arch.lpage_info[i - 1][0].write_count = 1;
7234                 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
7235                         slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
7236                 ugfn = slot->userspace_addr >> PAGE_SHIFT;
7237                 /*
7238                  * If the gfn and userspace address are not aligned wrt each
7239                  * other, or if explicitly asked to, disable large page
7240                  * support for this slot
7241                  */
7242                 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
7243                     !kvm_largepages_enabled()) {
7244                         unsigned long j;
7245
7246                         for (j = 0; j < lpages; ++j)
7247                                 slot->arch.lpage_info[i - 1][j].write_count = 1;
7248                 }
7249         }
7250
7251         return 0;
7252
7253 out_free:
7254         for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
7255                 kvm_kvfree(slot->arch.rmap[i]);
7256                 slot->arch.rmap[i] = NULL;
7257                 if (i == 0)
7258                         continue;
7259
7260                 kvm_kvfree(slot->arch.lpage_info[i - 1]);
7261                 slot->arch.lpage_info[i - 1] = NULL;
7262         }
7263         return -ENOMEM;
7264 }
7265
7266 void kvm_arch_memslots_updated(struct kvm *kvm)
7267 {
7268         /*
7269          * memslots->generation has been incremented.
7270          * mmio generation may have reached its maximum value.
7271          */
7272         kvm_mmu_invalidate_mmio_sptes(kvm);
7273 }
7274
7275 int kvm_arch_prepare_memory_region(struct kvm *kvm,
7276                                 struct kvm_memory_slot *memslot,
7277                                 struct kvm_userspace_memory_region *mem,
7278                                 enum kvm_mr_change change)
7279 {
7280         /*
7281          * Only private memory slots need to be mapped here since
7282          * KVM_SET_MEMORY_REGION ioctl is no longer supported.
7283          */
7284         if ((memslot->id >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_CREATE)) {
7285                 unsigned long userspace_addr;
7286
7287                 /*
7288                  * MAP_SHARED to prevent internal slot pages from being moved
7289                  * by fork()/COW.
7290                  */
7291                 userspace_addr = vm_mmap(NULL, 0, memslot->npages * PAGE_SIZE,
7292                                          PROT_READ | PROT_WRITE,
7293                                          MAP_SHARED | MAP_ANONYMOUS, 0);
7294
7295                 if (IS_ERR((void *)userspace_addr))
7296                         return PTR_ERR((void *)userspace_addr);
7297
7298                 memslot->userspace_addr = userspace_addr;
7299         }
7300
7301         return 0;
7302 }
7303
7304 void kvm_arch_commit_memory_region(struct kvm *kvm,
7305                                 struct kvm_userspace_memory_region *mem,
7306                                 const struct kvm_memory_slot *old,
7307                                 enum kvm_mr_change change)
7308 {
7309
7310         int nr_mmu_pages = 0;
7311
7312         if ((mem->slot >= KVM_USER_MEM_SLOTS) && (change == KVM_MR_DELETE)) {
7313                 int ret;
7314
7315                 ret = vm_munmap(old->userspace_addr,
7316                                 old->npages * PAGE_SIZE);
7317                 if (ret < 0)
7318                         printk(KERN_WARNING
7319                                "kvm_vm_ioctl_set_memory_region: "
7320                                "failed to munmap memory\n");
7321         }
7322
7323         if (!kvm->arch.n_requested_mmu_pages)
7324                 nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
7325
7326         if (nr_mmu_pages)
7327                 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
7328         /*
7329          * Write protect all pages for dirty logging.
7330          *
7331          * All the sptes including the large sptes which point to this
7332          * slot are set to readonly. We can not create any new large
7333          * spte on this slot until the end of the logging.
7334          *
7335          * See the comments in fast_page_fault().
7336          */
7337         if ((change != KVM_MR_DELETE) && (mem->flags & KVM_MEM_LOG_DIRTY_PAGES))
7338                 kvm_mmu_slot_remove_write_access(kvm, mem->slot);
7339 }
7340
7341 void kvm_arch_flush_shadow_all(struct kvm *kvm)
7342 {
7343         kvm_mmu_invalidate_zap_all_pages(kvm);
7344 }
7345
7346 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
7347                                    struct kvm_memory_slot *slot)
7348 {
7349         kvm_mmu_invalidate_zap_all_pages(kvm);
7350 }
7351
7352 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
7353 {
7354         if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events)
7355                 kvm_x86_ops->check_nested_events(vcpu, false);
7356
7357         return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
7358                 !vcpu->arch.apf.halted)
7359                 || !list_empty_careful(&vcpu->async_pf.done)
7360                 || kvm_apic_has_events(vcpu)
7361                 || vcpu->arch.pv.pv_unhalted
7362                 || atomic_read(&vcpu->arch.nmi_queued) ||
7363                 (kvm_arch_interrupt_allowed(vcpu) &&
7364                  kvm_cpu_has_interrupt(vcpu));
7365 }
7366
7367 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
7368 {
7369         return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
7370 }
7371
7372 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
7373 {
7374         return kvm_x86_ops->interrupt_allowed(vcpu);
7375 }
7376
7377 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
7378 {
7379         unsigned long current_rip = kvm_rip_read(vcpu) +
7380                 get_segment_base(vcpu, VCPU_SREG_CS);
7381
7382         return current_rip == linear_rip;
7383 }
7384 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
7385
7386 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
7387 {
7388         unsigned long rflags;
7389
7390         rflags = kvm_x86_ops->get_rflags(vcpu);
7391         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
7392                 rflags &= ~X86_EFLAGS_TF;
7393         return rflags;
7394 }
7395 EXPORT_SYMBOL_GPL(kvm_get_rflags);
7396
7397 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
7398 {
7399         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
7400             kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
7401                 rflags |= X86_EFLAGS_TF;
7402         kvm_x86_ops->set_rflags(vcpu, rflags);
7403         kvm_make_request(KVM_REQ_EVENT, vcpu);
7404 }
7405 EXPORT_SYMBOL_GPL(kvm_set_rflags);
7406
7407 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
7408 {
7409         int r;
7410
7411         if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) ||
7412               work->wakeup_all)
7413                 return;
7414
7415         r = kvm_mmu_reload(vcpu);
7416         if (unlikely(r))
7417                 return;
7418
7419         if (!vcpu->arch.mmu.direct_map &&
7420               work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu))
7421                 return;
7422
7423         vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true);
7424 }
7425
7426 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
7427 {
7428         return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
7429 }
7430
7431 static inline u32 kvm_async_pf_next_probe(u32 key)
7432 {
7433         return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1);
7434 }
7435
7436 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
7437 {
7438         u32 key = kvm_async_pf_hash_fn(gfn);
7439
7440         while (vcpu->arch.apf.gfns[key] != ~0)
7441                 key = kvm_async_pf_next_probe(key);
7442
7443         vcpu->arch.apf.gfns[key] = gfn;
7444 }
7445
7446 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
7447 {
7448         int i;
7449         u32 key = kvm_async_pf_hash_fn(gfn);
7450
7451         for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) &&
7452                      (vcpu->arch.apf.gfns[key] != gfn &&
7453                       vcpu->arch.apf.gfns[key] != ~0); i++)
7454                 key = kvm_async_pf_next_probe(key);
7455
7456         return key;
7457 }
7458
7459 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
7460 {
7461         return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
7462 }
7463
7464 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
7465 {
7466         u32 i, j, k;
7467
7468         i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
7469         while (true) {
7470                 vcpu->arch.apf.gfns[i] = ~0;
7471                 do {
7472                         j = kvm_async_pf_next_probe(j);
7473                         if (vcpu->arch.apf.gfns[j] == ~0)
7474                                 return;
7475                         k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
7476                         /*
7477                          * k lies cyclically in ]i,j]
7478                          * |    i.k.j |
7479                          * |....j i.k.| or  |.k..j i...|
7480                          */
7481                 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
7482                 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
7483                 i = j;
7484         }
7485 }
7486
7487 static int apf_put_user(struct kvm_vcpu *vcpu, u32 val)
7488 {
7489
7490         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val,
7491                                       sizeof(val));
7492 }
7493
7494 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
7495                                      struct kvm_async_pf *work)
7496 {
7497         struct x86_exception fault;
7498
7499         trace_kvm_async_pf_not_present(work->arch.token, work->gva);
7500         kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
7501
7502         if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) ||
7503             (vcpu->arch.apf.send_user_only &&
7504              kvm_x86_ops->get_cpl(vcpu) == 0))
7505                 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
7506         else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) {
7507                 fault.vector = PF_VECTOR;
7508                 fault.error_code_valid = true;
7509                 fault.error_code = 0;
7510                 fault.nested_page_fault = false;
7511                 fault.address = work->arch.token;
7512                 kvm_inject_page_fault(vcpu, &fault);
7513         }
7514 }
7515
7516 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
7517                                  struct kvm_async_pf *work)
7518 {
7519         struct x86_exception fault;
7520
7521         trace_kvm_async_pf_ready(work->arch.token, work->gva);
7522         if (work->wakeup_all)
7523                 work->arch.token = ~0; /* broadcast wakeup */
7524         else
7525                 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
7526
7527         if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) &&
7528             !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) {
7529                 fault.vector = PF_VECTOR;
7530                 fault.error_code_valid = true;
7531                 fault.error_code = 0;
7532                 fault.nested_page_fault = false;
7533                 fault.address = work->arch.token;
7534                 kvm_inject_page_fault(vcpu, &fault);
7535         }
7536         vcpu->arch.apf.halted = false;
7537         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
7538 }
7539
7540 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
7541 {
7542         if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED))
7543                 return true;
7544         else
7545                 return !kvm_event_needs_reinjection(vcpu) &&
7546                         kvm_x86_ops->interrupt_allowed(vcpu);
7547 }
7548
7549 void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
7550 {
7551         atomic_inc(&kvm->arch.noncoherent_dma_count);
7552 }
7553 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
7554
7555 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
7556 {
7557         atomic_dec(&kvm->arch.noncoherent_dma_count);
7558 }
7559 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
7560
7561 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
7562 {
7563         return atomic_read(&kvm->arch.noncoherent_dma_count);
7564 }
7565 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
7566
7567 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
7568 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
7569 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
7570 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
7571 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
7572 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
7573 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
7574 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
7575 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
7576 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
7577 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
7578 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
7579 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);