arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/slab.h>
  26 #include <linux/timer.h>
  27 #include <asm/asm-offsets.h>
  28 #include <asm/lowcore.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/nmi.h>
  31 #include <asm/switch_to.h>
  32 #include <asm/facility.h>
  33 #include <asm/sclp.h>
  34 #include "kvm-s390.h"
  35 #include "gaccess.h"
  36
  37 #define CREATE_TRACE_POINTS
  38 #include "trace.h"
  39 #include "trace-s390.h"
  40
  41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  42
  43 struct kvm_stats_debugfs_item debugfs_entries[] = {
  44         { "userspace_handled", VCPU_STAT(exit_userspace) },
  45         { "exit_null", VCPU_STAT(exit_null) },
  46         { "exit_validity", VCPU_STAT(exit_validity) },
  47         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  48         { "exit_external_request", VCPU_STAT(exit_external_request) },
  49         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  50         { "exit_instruction", VCPU_STAT(exit_instruction) },
  51         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  52         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  53         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  54         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  55         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  56         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  57         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  58         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  59         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  60         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  61         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  62         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  63         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  64         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  65         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  66         { "instruction_spx", VCPU_STAT(instruction_spx) },
  67         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  68         { "instruction_stap", VCPU_STAT(instruction_stap) },
  69         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  70         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  71         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  72         { "instruction_essa", VCPU_STAT(instruction_essa) },
  73         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  74         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  75         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  76         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  77         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  78         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  79         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  80         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  81         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  82         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  83         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  84         { "diagnose_10", VCPU_STAT(diagnose_10) },
  85         { "diagnose_44", VCPU_STAT(diagnose_44) },
  86         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  87         { NULL }
  88 };
  89
  90 unsigned long *vfacilities;
  91 static struct gmap_notifier gmap_notifier;
  92
  93 /* test availability of vfacility */
  94 int test_vfacility(unsigned long nr)
  95 {
  96         return __test_facility(nr, (void *) vfacilities);
  97 }
  98
  99 /* Section: not file related */
 100 int kvm_arch_hardware_enable(void *garbage)
 101 {
 102         /* every s390 is virtualization enabled ;-) */
 103         return 0;
 104 }
 105
 106 void kvm_arch_hardware_disable(void *garbage)
 107 {
 108 }
 109
 110 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 111
 112 int kvm_arch_hardware_setup(void)
 113 {
 114         gmap_notifier.notifier_call = kvm_gmap_notifier;
 115         gmap_register_ipte_notifier(&gmap_notifier);
 116         return 0;
 117 }
 118
 119 void kvm_arch_hardware_unsetup(void)
 120 {
 121         gmap_unregister_ipte_notifier(&gmap_notifier);
 122 }
 123
 124 void kvm_arch_check_processor_compat(void *rtn)
 125 {
 126 }
 127
 128 int kvm_arch_init(void *opaque)
 129 {
 130         return 0;
 131 }
 132
 133 void kvm_arch_exit(void)
 134 {
 135 }
 136
 137 /* Section: device related */
 138 long kvm_arch_dev_ioctl(struct file *filp,
 139                         unsigned int ioctl, unsigned long arg)
 140 {
 141         if (ioctl == KVM_S390_ENABLE_SIE)
 142                 return s390_enable_sie();
 143         return -EINVAL;
 144 }
 145
 146 int kvm_dev_ioctl_check_extension(long ext)
 147 {
 148         int r;
 149
 150         switch (ext) {
 151         case KVM_CAP_S390_PSW:
 152         case KVM_CAP_S390_GMAP:
 153         case KVM_CAP_SYNC_MMU:
 154 #ifdef CONFIG_KVM_S390_UCONTROL
 155         case KVM_CAP_S390_UCONTROL:
 156 #endif
 157         case KVM_CAP_ASYNC_PF:
 158         case KVM_CAP_SYNC_REGS:
 159         case KVM_CAP_ONE_REG:
 160         case KVM_CAP_ENABLE_CAP:
 161         case KVM_CAP_S390_CSS_SUPPORT:
 162         case KVM_CAP_IOEVENTFD:
 163         case KVM_CAP_DEVICE_CTRL:
 164         case KVM_CAP_ENABLE_CAP_VM:
 165         case KVM_CAP_VM_ATTRIBUTES:
 166                 r = 1;
 167                 break;
 168         case KVM_CAP_NR_VCPUS:
 169         case KVM_CAP_MAX_VCPUS:
 170                 r = KVM_MAX_VCPUS;
 171                 break;
 172         case KVM_CAP_NR_MEMSLOTS:
 173                 r = KVM_USER_MEM_SLOTS;
 174                 break;
 175         case KVM_CAP_S390_COW:
 176                 r = MACHINE_HAS_ESOP;
 177                 break;
 178         default:
 179                 r = 0;
 180         }
 181         return r;
 182 }
 183
 184 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 185                                         struct kvm_memory_slot *memslot)
 186 {
 187         gfn_t cur_gfn, last_gfn;
 188         unsigned long address;
 189         struct gmap *gmap = kvm->arch.gmap;
 190
 191         down_read(&gmap->mm->mmap_sem);
 192         /* Loop over all guest pages */
 193         last_gfn = memslot->base_gfn + memslot->npages;
 194         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 195                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 196
 197                 if (gmap_test_and_clear_dirty(address, gmap))
 198                         mark_page_dirty(kvm, cur_gfn);
 199         }
 200         up_read(&gmap->mm->mmap_sem);
 201 }
 202
 203 /* Section: vm related */
 204 /*
 205  * Get (and clear) the dirty memory log for a memory slot.
 206  */
 207 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 208                                struct kvm_dirty_log *log)
 209 {
 210         int r;
 211         unsigned long n;
 212         struct kvm_memory_slot *memslot;
 213         int is_dirty = 0;
 214
 215         mutex_lock(&kvm->slots_lock);
 216
 217         r = -EINVAL;
 218         if (log->slot >= KVM_USER_MEM_SLOTS)
 219                 goto out;
 220
 221         memslot = id_to_memslot(kvm->memslots, log->slot);
 222         r = -ENOENT;
 223         if (!memslot->dirty_bitmap)
 224                 goto out;
 225
 226         kvm_s390_sync_dirty_log(kvm, memslot);
 227         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 228         if (r)
 229                 goto out;
 230
 231         /* Clear the dirty log */
 232         if (is_dirty) {
 233                 n = kvm_dirty_bitmap_bytes(memslot);
 234                 memset(memslot->dirty_bitmap, 0, n);
 235         }
 236         r = 0;
 237 out:
 238         mutex_unlock(&kvm->slots_lock);
 239         return r;
 240 }
 241
 242 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 243 {
 244         int r;
 245
 246         if (cap->flags)
 247                 return -EINVAL;
 248
 249         switch (cap->cap) {
 250         case KVM_CAP_S390_IRQCHIP:
 251                 kvm->arch.use_irqchip = 1;
 252                 r = 0;
 253                 break;
 254         default:
 255                 r = -EINVAL;
 256                 break;
 257         }
 258         return r;
 259 }
 260
 261 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 262 {
 263         int ret;
 264         unsigned int idx;
 265         switch (attr->attr) {
 266         case KVM_S390_VM_MEM_ENABLE_CMMA:
 267                 ret = -EBUSY;
 268                 mutex_lock(&kvm->lock);
 269                 if (atomic_read(&kvm->online_vcpus) == 0) {
 270                         kvm->arch.use_cmma = 1;
 271                         ret = 0;
 272                 }
 273                 mutex_unlock(&kvm->lock);
 274                 break;
 275         case KVM_S390_VM_MEM_CLR_CMMA:
 276                 mutex_lock(&kvm->lock);
 277                 idx = srcu_read_lock(&kvm->srcu);
 278                 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
 279                 srcu_read_unlock(&kvm->srcu, idx);
 280                 mutex_unlock(&kvm->lock);
 281                 ret = 0;
 282                 break;
 283         default:
 284                 ret = -ENXIO;
 285                 break;
 286         }
 287         return ret;
 288 }
 289
 290 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 291 {
 292         int ret;
 293
 294         switch (attr->group) {
 295         case KVM_S390_VM_MEM_CTRL:
 296                 ret = kvm_s390_mem_control(kvm, attr);
 297                 break;
 298         default:
 299                 ret = -ENXIO;
 300                 break;
 301         }
 302
 303         return ret;
 304 }
 305
 306 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 307 {
 308         return -ENXIO;
 309 }
 310
 311 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 312 {
 313         int ret;
 314
 315         switch (attr->group) {
 316         case KVM_S390_VM_MEM_CTRL:
 317                 switch (attr->attr) {
 318                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 319                 case KVM_S390_VM_MEM_CLR_CMMA:
 320                         ret = 0;
 321                         break;
 322                 default:
 323                         ret = -ENXIO;
 324                         break;
 325                 }
 326                 break;
 327         default:
 328                 ret = -ENXIO;
 329                 break;
 330         }
 331
 332         return ret;
 333 }
 334
 335 long kvm_arch_vm_ioctl(struct file *filp,
 336                        unsigned int ioctl, unsigned long arg)
 337 {
 338         struct kvm *kvm = filp->private_data;
 339         void __user *argp = (void __user *)arg;
 340         struct kvm_device_attr attr;
 341         int r;
 342
 343         switch (ioctl) {
 344         case KVM_S390_INTERRUPT: {
 345                 struct kvm_s390_interrupt s390int;
 346
 347                 r = -EFAULT;
 348                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 349                         break;
 350                 r = kvm_s390_inject_vm(kvm, &s390int);
 351                 break;
 352         }
 353         case KVM_ENABLE_CAP: {
 354                 struct kvm_enable_cap cap;
 355                 r = -EFAULT;
 356                 if (copy_from_user(&cap, argp, sizeof(cap)))
 357                         break;
 358                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 359                 break;
 360         }
 361         case KVM_CREATE_IRQCHIP: {
 362                 struct kvm_irq_routing_entry routing;
 363
 364                 r = -EINVAL;
 365                 if (kvm->arch.use_irqchip) {
 366                         /* Set up dummy routing. */
 367                         memset(&routing, 0, sizeof(routing));
 368                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 369                         r = 0;
 370                 }
 371                 break;
 372         }
 373         case KVM_SET_DEVICE_ATTR: {
 374                 r = -EFAULT;
 375                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 376                         break;
 377                 r = kvm_s390_vm_set_attr(kvm, &attr);
 378                 break;
 379         }
 380         case KVM_GET_DEVICE_ATTR: {
 381                 r = -EFAULT;
 382                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 383                         break;
 384                 r = kvm_s390_vm_get_attr(kvm, &attr);
 385                 break;
 386         }
 387         case KVM_HAS_DEVICE_ATTR: {
 388                 r = -EFAULT;
 389                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 390                         break;
 391                 r = kvm_s390_vm_has_attr(kvm, &attr);
 392                 break;
 393         }
 394         default:
 395                 r = -ENOTTY;
 396         }
 397
 398         return r;
 399 }
 400
 401 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 402 {
 403         int rc;
 404         char debug_name[16];
 405         static unsigned long sca_offset;
 406
 407         rc = -EINVAL;
 408 #ifdef CONFIG_KVM_S390_UCONTROL
 409         if (type & ~KVM_VM_S390_UCONTROL)
 410                 goto out_err;
 411         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 412                 goto out_err;
 413 #else
 414         if (type)
 415                 goto out_err;
 416 #endif
 417
 418         rc = s390_enable_sie();
 419         if (rc)
 420                 goto out_err;
 421
 422         rc = -ENOMEM;
 423
 424         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 425         if (!kvm->arch.sca)
 426                 goto out_err;
 427         spin_lock(&kvm_lock);
 428         sca_offset = (sca_offset + 16) & 0x7f0;
 429         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 430         spin_unlock(&kvm_lock);
 431
 432         sprintf(debug_name, "kvm-%u", current->pid);
 433
 434         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 435         if (!kvm->arch.dbf)
 436                 goto out_nodbf;
 437
 438         spin_lock_init(&kvm->arch.float_int.lock);
 439         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 440
 441         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 442         VM_EVENT(kvm, 3, "%s", "vm created");
 443
 444         if (type & KVM_VM_S390_UCONTROL) {
 445                 kvm->arch.gmap = NULL;
 446         } else {
 447                 kvm->arch.gmap = gmap_alloc(current->mm);
 448                 if (!kvm->arch.gmap)
 449                         goto out_nogmap;
 450                 kvm->arch.gmap->private = kvm;
 451                 kvm->arch.gmap->pfault_enabled = 0;
 452         }
 453
 454         kvm->arch.css_support = 0;
 455         kvm->arch.use_irqchip = 0;
 456
 457         return 0;
 458 out_nogmap:
 459         debug_unregister(kvm->arch.dbf);
 460 out_nodbf:
 461         free_page((unsigned long)(kvm->arch.sca));
 462 out_err:
 463         return rc;
 464 }
 465
 466 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 467 {
 468         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 469         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 470         kvm_clear_async_pf_completion_queue(vcpu);
 471         if (!kvm_is_ucontrol(vcpu->kvm)) {
 472                 clear_bit(63 - vcpu->vcpu_id,
 473                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 474                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 475                     (__u64) vcpu->arch.sie_block)
 476                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 477         }
 478         smp_mb();
 479
 480         if (kvm_is_ucontrol(vcpu->kvm))
 481                 gmap_free(vcpu->arch.gmap);
 482
 483         if (kvm_s390_cmma_enabled(vcpu->kvm))
 484                 kvm_s390_vcpu_unsetup_cmma(vcpu);
 485         free_page((unsigned long)(vcpu->arch.sie_block));
 486
 487         kvm_vcpu_uninit(vcpu);
 488         kmem_cache_free(kvm_vcpu_cache, vcpu);
 489 }
 490
 491 static void kvm_free_vcpus(struct kvm *kvm)
 492 {
 493         unsigned int i;
 494         struct kvm_vcpu *vcpu;
 495
 496         kvm_for_each_vcpu(i, vcpu, kvm)
 497                 kvm_arch_vcpu_destroy(vcpu);
 498
 499         mutex_lock(&kvm->lock);
 500         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 501                 kvm->vcpus[i] = NULL;
 502
 503         atomic_set(&kvm->online_vcpus, 0);
 504         mutex_unlock(&kvm->lock);
 505 }
 506
 507 void kvm_arch_sync_events(struct kvm *kvm)
 508 {
 509 }
 510
 511 void kvm_arch_destroy_vm(struct kvm *kvm)
 512 {
 513         kvm_free_vcpus(kvm);
 514         free_page((unsigned long)(kvm->arch.sca));
 515         debug_unregister(kvm->arch.dbf);
 516         if (!kvm_is_ucontrol(kvm))
 517                 gmap_free(kvm->arch.gmap);
 518         kvm_s390_destroy_adapters(kvm);
 519 }
 520
 521 /* Section: vcpu related */
 522 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 523 {
 524         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 525         kvm_clear_async_pf_completion_queue(vcpu);
 526         if (kvm_is_ucontrol(vcpu->kvm)) {
 527                 vcpu->arch.gmap = gmap_alloc(current->mm);
 528                 if (!vcpu->arch.gmap)
 529                         return -ENOMEM;
 530                 vcpu->arch.gmap->private = vcpu->kvm;
 531                 return 0;
 532         }
 533
 534         vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 535         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
 536                                     KVM_SYNC_GPRS |
 537                                     KVM_SYNC_ACRS |
 538                                     KVM_SYNC_CRS;
 539         return 0;
 540 }
 541
 542 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 543 {
 544         /* Nothing todo */
 545 }
 546
 547 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 548 {
 549         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 550         save_fp_regs(vcpu->arch.host_fpregs.fprs);
 551         save_access_regs(vcpu->arch.host_acrs);
 552         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 553         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 554         restore_access_regs(vcpu->run->s.regs.acrs);
 555         gmap_enable(vcpu->arch.gmap);
 556         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 557 }
 558
 559 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 560 {
 561         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 562         gmap_disable(vcpu->arch.gmap);
 563         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 564         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
 565         save_access_regs(vcpu->run->s.regs.acrs);
 566         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 567         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
 568         restore_access_regs(vcpu->arch.host_acrs);
 569 }
 570
 571 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 572 {
 573         /* this equals initial cpu reset in pop, but we don't switch to ESA */
 574         vcpu->arch.sie_block->gpsw.mask = 0UL;
 575         vcpu->arch.sie_block->gpsw.addr = 0UL;
 576         kvm_s390_set_prefix(vcpu, 0);
 577         vcpu->arch.sie_block->cputm     = 0UL;
 578         vcpu->arch.sie_block->ckc       = 0UL;
 579         vcpu->arch.sie_block->todpr     = 0;
 580         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 581         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 582         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 583         vcpu->arch.guest_fpregs.fpc = 0;
 584         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 585         vcpu->arch.sie_block->gbea = 1;
 586         vcpu->arch.sie_block->pp = 0;
 587         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 588         kvm_clear_async_pf_completion_queue(vcpu);
 589         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
 590         kvm_s390_clear_local_irqs(vcpu);
 591 }
 592
 593 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 594 {
 595         return 0;
 596 }
 597
 598 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
 599 {
 600         free_page(vcpu->arch.sie_block->cbrlo);
 601         vcpu->arch.sie_block->cbrlo = 0;
 602 }
 603
 604 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
 605 {
 606         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
 607         if (!vcpu->arch.sie_block->cbrlo)
 608                 return -ENOMEM;
 609
 610         vcpu->arch.sie_block->ecb2 |= 0x80;
 611         vcpu->arch.sie_block->ecb2 &= ~0x08;
 612         return 0;
 613 }
 614
 615 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 616 {
 617         int rc = 0;
 618
 619         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
 620                                                     CPUSTAT_SM |
 621                                                     CPUSTAT_STOPPED |
 622                                                     CPUSTAT_GED);
 623         vcpu->arch.sie_block->ecb   = 6;
 624         if (test_vfacility(50) && test_vfacility(73))
 625                 vcpu->arch.sie_block->ecb |= 0x10;
 626
 627         vcpu->arch.sie_block->ecb2  = 8;
 628         vcpu->arch.sie_block->eca   = 0xC1002000U;
 629         if (sclp_has_siif())
 630                 vcpu->arch.sie_block->eca |= 1;
 631         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 632         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
 633         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
 634                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
 635                 if (rc)
 636                         return rc;
 637         }
 638         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 639         tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 640                      (unsigned long) vcpu);
 641         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 642         get_cpu_id(&vcpu->arch.cpu_id);
 643         vcpu->arch.cpu_id.version = 0xff;
 644         return rc;
 645 }
 646
 647 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 648                                       unsigned int id)
 649 {
 650         struct kvm_vcpu *vcpu;
 651         struct sie_page *sie_page;
 652         int rc = -EINVAL;
 653
 654         if (id >= KVM_MAX_VCPUS)
 655                 goto out;
 656
 657         rc = -ENOMEM;
 658
 659         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 660         if (!vcpu)
 661                 goto out;
 662
 663         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
 664         if (!sie_page)
 665                 goto out_free_cpu;
 666
 667         vcpu->arch.sie_block = &sie_page->sie_block;
 668         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
 669
 670         vcpu->arch.sie_block->icpua = id;
 671         if (!kvm_is_ucontrol(kvm)) {
 672                 if (!kvm->arch.sca) {
 673                         WARN_ON_ONCE(1);
 674                         goto out_free_cpu;
 675                 }
 676                 if (!kvm->arch.sca->cpu[id].sda)
 677                         kvm->arch.sca->cpu[id].sda =
 678                                 (__u64) vcpu->arch.sie_block;
 679                 vcpu->arch.sie_block->scaoh =
 680                         (__u32)(((__u64)kvm->arch.sca) >> 32);
 681                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 682                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 683         }
 684
 685         spin_lock_init(&vcpu->arch.local_int.lock);
 686         INIT_LIST_HEAD(&vcpu->arch.local_int.list);
 687         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 688         vcpu->arch.local_int.wq = &vcpu->wq;
 689         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 690
 691         rc = kvm_vcpu_init(vcpu, kvm, id);
 692         if (rc)
 693                 goto out_free_sie_block;
 694         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 695                  vcpu->arch.sie_block);
 696         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 697
 698         return vcpu;
 699 out_free_sie_block:
 700         free_page((unsigned long)(vcpu->arch.sie_block));
 701 out_free_cpu:
 702         kmem_cache_free(kvm_vcpu_cache, vcpu);
 703 out:
 704         return ERR_PTR(rc);
 705 }
 706
 707 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 708 {
 709         return kvm_cpu_has_interrupt(vcpu);
 710 }
 711
 712 void s390_vcpu_block(struct kvm_vcpu *vcpu)
 713 {
 714         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 715 }
 716
 717 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 718 {
 719         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 720 }
 721
 722 /*
 723  * Kick a guest cpu out of SIE and wait until SIE is not running.
 724  * If the CPU is not running (e.g. waiting as idle) the function will
 725  * return immediately. */
 726 void exit_sie(struct kvm_vcpu *vcpu)
 727 {
 728         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
 729         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 730                 cpu_relax();
 731 }
 732
 733 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
 734 void exit_sie_sync(struct kvm_vcpu *vcpu)
 735 {
 736         s390_vcpu_block(vcpu);
 737         exit_sie(vcpu);
 738 }
 739
 740 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 741 {
 742         int i;
 743         struct kvm *kvm = gmap->private;
 744         struct kvm_vcpu *vcpu;
 745
 746         kvm_for_each_vcpu(i, vcpu, kvm) {
 747                 /* match against both prefix pages */
 748                 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
 749                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 750                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 751                         exit_sie_sync(vcpu);
 752                 }
 753         }
 754 }
 755
 756 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 757 {
 758         /* kvm common code refers to this, but never calls it */
 759         BUG();
 760         return 0;
 761 }
 762
 763 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
 764                                            struct kvm_one_reg *reg)
 765 {
 766         int r = -EINVAL;
 767
 768         switch (reg->id) {
 769         case KVM_REG_S390_TODPR:
 770                 r = put_user(vcpu->arch.sie_block->todpr,
 771                              (u32 __user *)reg->addr);
 772                 break;
 773         case KVM_REG_S390_EPOCHDIFF:
 774                 r = put_user(vcpu->arch.sie_block->epoch,
 775                              (u64 __user *)reg->addr);
 776                 break;
 777         case KVM_REG_S390_CPU_TIMER:
 778                 r = put_user(vcpu->arch.sie_block->cputm,
 779                              (u64 __user *)reg->addr);
 780                 break;
 781         case KVM_REG_S390_CLOCK_COMP:
 782                 r = put_user(vcpu->arch.sie_block->ckc,
 783                              (u64 __user *)reg->addr);
 784                 break;
 785         case KVM_REG_S390_PFTOKEN:
 786                 r = put_user(vcpu->arch.pfault_token,
 787                              (u64 __user *)reg->addr);
 788                 break;
 789         case KVM_REG_S390_PFCOMPARE:
 790                 r = put_user(vcpu->arch.pfault_compare,
 791                              (u64 __user *)reg->addr);
 792                 break;
 793         case KVM_REG_S390_PFSELECT:
 794                 r = put_user(vcpu->arch.pfault_select,
 795                              (u64 __user *)reg->addr);
 796                 break;
 797         case KVM_REG_S390_PP:
 798                 r = put_user(vcpu->arch.sie_block->pp,
 799                              (u64 __user *)reg->addr);
 800                 break;
 801         case KVM_REG_S390_GBEA:
 802                 r = put_user(vcpu->arch.sie_block->gbea,
 803                              (u64 __user *)reg->addr);
 804                 break;
 805         default:
 806                 break;
 807         }
 808
 809         return r;
 810 }
 811
 812 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 813                                            struct kvm_one_reg *reg)
 814 {
 815         int r = -EINVAL;
 816
 817         switch (reg->id) {
 818         case KVM_REG_S390_TODPR:
 819                 r = get_user(vcpu->arch.sie_block->todpr,
 820                              (u32 __user *)reg->addr);
 821                 break;
 822         case KVM_REG_S390_EPOCHDIFF:
 823                 r = get_user(vcpu->arch.sie_block->epoch,
 824                              (u64 __user *)reg->addr);
 825                 break;
 826         case KVM_REG_S390_CPU_TIMER:
 827                 r = get_user(vcpu->arch.sie_block->cputm,
 828                              (u64 __user *)reg->addr);
 829                 break;
 830         case KVM_REG_S390_CLOCK_COMP:
 831                 r = get_user(vcpu->arch.sie_block->ckc,
 832                              (u64 __user *)reg->addr);
 833                 break;
 834         case KVM_REG_S390_PFTOKEN:
 835                 r = get_user(vcpu->arch.pfault_token,
 836                              (u64 __user *)reg->addr);
 837                 break;
 838         case KVM_REG_S390_PFCOMPARE:
 839                 r = get_user(vcpu->arch.pfault_compare,
 840                              (u64 __user *)reg->addr);
 841                 break;
 842         case KVM_REG_S390_PFSELECT:
 843                 r = get_user(vcpu->arch.pfault_select,
 844                              (u64 __user *)reg->addr);
 845                 break;
 846         case KVM_REG_S390_PP:
 847                 r = get_user(vcpu->arch.sie_block->pp,
 848                              (u64 __user *)reg->addr);
 849                 break;
 850         case KVM_REG_S390_GBEA:
 851                 r = get_user(vcpu->arch.sie_block->gbea,
 852                              (u64 __user *)reg->addr);
 853                 break;
 854         default:
 855                 break;
 856         }
 857
 858         return r;
 859 }
 860
 861 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 862 {
 863         kvm_s390_vcpu_initial_reset(vcpu);
 864         return 0;
 865 }
 866
 867 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 868 {
 869         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
 870         return 0;
 871 }
 872
 873 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 874 {
 875         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
 876         return 0;
 877 }
 878
 879 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 880                                   struct kvm_sregs *sregs)
 881 {
 882         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 883         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
 884         restore_access_regs(vcpu->run->s.regs.acrs);
 885         return 0;
 886 }
 887
 888 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 889                                   struct kvm_sregs *sregs)
 890 {
 891         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 892         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
 893         return 0;
 894 }
 895
 896 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 897 {
 898         if (test_fp_ctl(fpu->fpc))
 899                 return -EINVAL;
 900         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 901         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
 902         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 903         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 904         return 0;
 905 }
 906
 907 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 908 {
 909         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 910         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
 911         return 0;
 912 }
 913
 914 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 915 {
 916         int rc = 0;
 917
 918         if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
 919                 rc = -EBUSY;
 920         else {
 921                 vcpu->run->psw_mask = psw.mask;
 922                 vcpu->run->psw_addr = psw.addr;
 923         }
 924         return rc;
 925 }
 926
 927 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 928                                   struct kvm_translation *tr)
 929 {
 930         return -EINVAL; /* not implemented yet */
 931 }
 932
 933 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 934                                         struct kvm_guest_debug *dbg)
 935 {
 936         return -EINVAL; /* not implemented yet */
 937 }
 938
 939 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 940                                     struct kvm_mp_state *mp_state)
 941 {
 942         return -EINVAL; /* not implemented yet */
 943 }
 944
 945 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 946                                     struct kvm_mp_state *mp_state)
 947 {
 948         return -EINVAL; /* not implemented yet */
 949 }
 950
 951 bool kvm_s390_cmma_enabled(struct kvm *kvm)
 952 {
 953         if (!MACHINE_IS_LPAR)
 954                 return false;
 955         /* only enable for z10 and later */
 956         if (!MACHINE_HAS_EDAT1)
 957                 return false;
 958         if (!kvm->arch.use_cmma)
 959                 return false;
 960         return true;
 961 }
 962
 963 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 964 {
 965         /*
 966          * We use MMU_RELOAD just to re-arm the ipte notifier for the
 967          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
 968          * This ensures that the ipte instruction for this request has
 969          * already finished. We might race against a second unmapper that
 970          * wants to set the blocking bit. Lets just retry the request loop.
 971          */
 972         while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
 973                 int rc;
 974                 rc = gmap_ipte_notify(vcpu->arch.gmap,
 975                                       vcpu->arch.sie_block->prefix,
 976                                       PAGE_SIZE * 2);
 977                 if (rc)
 978                         return rc;
 979                 s390_vcpu_unblock(vcpu);
 980         }
 981         return 0;
 982 }
 983
 984 static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
 985 {
 986         long rc;
 987         hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
 988         struct mm_struct *mm = current->mm;
 989         down_read(&mm->mmap_sem);
 990         rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
 991         up_read(&mm->mmap_sem);
 992         return rc;
 993 }
 994
 995 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
 996                                       unsigned long token)
 997 {
 998         struct kvm_s390_interrupt inti;
 999         inti.parm64 = token;
1000
1001         if (start_token) {
1002                 inti.type = KVM_S390_INT_PFAULT_INIT;
1003                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1004         } else {
1005                 inti.type = KVM_S390_INT_PFAULT_DONE;
1006                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1007         }
1008 }
1009
1010 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1011                                      struct kvm_async_pf *work)
1012 {
1013         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1014         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1015 }
1016
1017 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1018                                  struct kvm_async_pf *work)
1019 {
1020         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1021         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1022 }
1023
1024 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1025                                struct kvm_async_pf *work)
1026 {
1027         /* s390 will always inject the page directly */
1028 }
1029
1030 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1031 {
1032         /*
1033          * s390 will always inject the page directly,
1034          * but we still want check_async_completion to cleanup
1035          */
1036         return true;
1037 }
1038
1039 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1040 {
1041         hva_t hva;
1042         struct kvm_arch_async_pf arch;
1043         int rc;
1044
1045         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1046                 return 0;
1047         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1048             vcpu->arch.pfault_compare)
1049                 return 0;
1050         if (psw_extint_disabled(vcpu))
1051                 return 0;
1052         if (kvm_cpu_has_interrupt(vcpu))
1053                 return 0;
1054         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1055                 return 0;
1056         if (!vcpu->arch.gmap->pfault_enabled)
1057                 return 0;
1058
1059         hva = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
1060         if (copy_from_guest(vcpu, &arch.pfault_token, vcpu->arch.pfault_token, 8))
1061                 return 0;
1062
1063         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1064         return rc;
1065 }
1066
1067 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1068 {
1069         int rc, cpuflags;
1070
1071         /*
1072          * On s390 notifications for arriving pages will be delivered directly
1073          * to the guest but the house keeping for completed pfaults is
1074          * handled outside the worker.
1075          */
1076         kvm_check_async_pf_completion(vcpu);
1077
1078         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1079
1080         if (need_resched())
1081                 schedule();
1082
1083         if (test_thread_flag(TIF_MCCK_PENDING))
1084                 s390_handle_mcck();
1085
1086         if (!kvm_is_ucontrol(vcpu->kvm))
1087                 kvm_s390_deliver_pending_interrupts(vcpu);
1088
1089         rc = kvm_s390_handle_requests(vcpu);
1090         if (rc)
1091                 return rc;
1092
1093         vcpu->arch.sie_block->icptcode = 0;
1094         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1095         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1096         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1097
1098         return 0;
1099 }
1100
1101 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1102 {
1103         int rc = -1;
1104
1105         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1106                    vcpu->arch.sie_block->icptcode);
1107         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1108
1109         if (exit_reason >= 0) {
1110                 rc = 0;
1111         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1112                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1113                 vcpu->run->s390_ucontrol.trans_exc_code =
1114                                                 current->thread.gmap_addr;
1115                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1116                 rc = -EREMOTE;
1117
1118         } else if (current->thread.gmap_pfault) {
1119                 trace_kvm_s390_major_guest_pfault(vcpu);
1120                 current->thread.gmap_pfault = 0;
1121                 if (kvm_arch_setup_async_pf(vcpu) ||
1122                     (kvm_arch_fault_in_sync(vcpu) >= 0))
1123                         rc = 0;
1124         }
1125
1126         if (rc == -1) {
1127                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1128                 trace_kvm_s390_sie_fault(vcpu);
1129                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1130         }
1131
1132         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1133
1134         if (rc == 0) {
1135                 if (kvm_is_ucontrol(vcpu->kvm))
1136                         /* Don't exit for host interrupts. */
1137                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1138                 else
1139                         rc = kvm_handle_sie_intercept(vcpu);
1140         }
1141
1142         return rc;
1143 }
1144
1145 static int __vcpu_run(struct kvm_vcpu *vcpu)
1146 {
1147         int rc, exit_reason;
1148
1149         /*
1150          * We try to hold kvm->srcu during most of vcpu_run (except when run-
1151          * ning the guest), so that memslots (and other stuff) are protected
1152          */
1153         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1154
1155         do {
1156                 rc = vcpu_pre_run(vcpu);
1157                 if (rc)
1158                         break;
1159
1160                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1161                 /*
1162                  * As PF_VCPU will be used in fault handler, between
1163                  * guest_enter and guest_exit should be no uaccess.
1164                  */
1165                 preempt_disable();
1166                 kvm_guest_enter();
1167                 preempt_enable();
1168                 exit_reason = sie64a(vcpu->arch.sie_block,
1169                                      vcpu->run->s.regs.gprs);
1170                 kvm_guest_exit();
1171                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1172
1173                 rc = vcpu_post_run(vcpu, exit_reason);
1174         } while (!signal_pending(current) && !rc);
1175
1176         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1177         return rc;
1178 }
1179
1180 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1181 {
1182         int rc;
1183         sigset_t sigsaved;
1184
1185         if (vcpu->sigset_active)
1186                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1187
1188         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1189
1190         switch (kvm_run->exit_reason) {
1191         case KVM_EXIT_S390_SIEIC:
1192         case KVM_EXIT_UNKNOWN:
1193         case KVM_EXIT_INTR:
1194         case KVM_EXIT_S390_RESET:
1195         case KVM_EXIT_S390_UCONTROL:
1196         case KVM_EXIT_S390_TSCH:
1197                 break;
1198         default:
1199                 BUG();
1200         }
1201
1202         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1203         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1204         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1205                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1206                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1207         }
1208         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1209                 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1210                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1211                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1212         }
1213
1214         might_fault();
1215         rc = __vcpu_run(vcpu);
1216
1217         if (signal_pending(current) && !rc) {
1218                 kvm_run->exit_reason = KVM_EXIT_INTR;
1219                 rc = -EINTR;
1220         }
1221
1222         if (rc == -EOPNOTSUPP) {
1223                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1224                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1225                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1226                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1227                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1228                 rc = 0;
1229         }
1230
1231         if (rc == -EREMOTE) {
1232                 /* intercept was handled, but userspace support is needed
1233                  * kvm_run has been prepared by the handler */
1234                 rc = 0;
1235         }
1236
1237         kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
1238         kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
1239         kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
1240         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1241
1242         if (vcpu->sigset_active)
1243                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1244
1245         vcpu->stat.exit_userspace++;
1246         return rc;
1247 }
1248
1249 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, void *from,
1250                        unsigned long n, int prefix)
1251 {
1252         if (prefix)
1253                 return copy_to_guest(vcpu, guestdest, from, n);
1254         else
1255                 return copy_to_guest_absolute(vcpu, guestdest, from, n);
1256 }
1257
1258 /*
1259  * store status at address
1260  * we use have two special cases:
1261  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1262  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1263  */
1264 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr)
1265 {
1266         unsigned char archmode = 1;
1267         int prefix;
1268         u64 clkcomp;
1269
1270         if (addr == KVM_S390_STORE_STATUS_NOADDR) {
1271                 if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1))
1272                         return -EFAULT;
1273                 addr = SAVE_AREA_BASE;
1274                 prefix = 0;
1275         } else if (addr == KVM_S390_STORE_STATUS_PREFIXED) {
1276                 if (copy_to_guest(vcpu, 163ul, &archmode, 1))
1277                         return -EFAULT;
1278                 addr = SAVE_AREA_BASE;
1279                 prefix = 1;
1280         } else
1281                 prefix = 0;
1282
1283         if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs),
1284                         vcpu->arch.guest_fpregs.fprs, 128, prefix))
1285                 return -EFAULT;
1286
1287         if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs),
1288                         vcpu->run->s.regs.gprs, 128, prefix))
1289                 return -EFAULT;
1290
1291         if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw),
1292                         &vcpu->arch.sie_block->gpsw, 16, prefix))
1293                 return -EFAULT;
1294
1295         if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg),
1296                         &vcpu->arch.sie_block->prefix, 4, prefix))
1297                 return -EFAULT;
1298
1299         if (__guestcopy(vcpu,
1300                         addr + offsetof(struct save_area, fp_ctrl_reg),
1301                         &vcpu->arch.guest_fpregs.fpc, 4, prefix))
1302                 return -EFAULT;
1303
1304         if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg),
1305                         &vcpu->arch.sie_block->todpr, 4, prefix))
1306                 return -EFAULT;
1307
1308         if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer),
1309                         &vcpu->arch.sie_block->cputm, 8, prefix))
1310                 return -EFAULT;
1311
1312         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1313         if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp),
1314                         &clkcomp, 8, prefix))
1315                 return -EFAULT;
1316
1317         if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs),
1318                         &vcpu->run->s.regs.acrs, 64, prefix))
1319                 return -EFAULT;
1320
1321         if (__guestcopy(vcpu,
1322                         addr + offsetof(struct save_area, ctrl_regs),
1323                         &vcpu->arch.sie_block->gcr, 128, prefix))
1324                 return -EFAULT;
1325         return 0;
1326 }
1327
1328 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1329 {
1330         /*
1331          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1332          * copying in vcpu load/put. Lets update our copies before we save
1333          * it into the save area
1334          */
1335         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1336         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1337         save_access_regs(vcpu->run->s.regs.acrs);
1338
1339         return kvm_s390_store_status_unloaded(vcpu, addr);
1340 }
1341
1342 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1343                                      struct kvm_enable_cap *cap)
1344 {
1345         int r;
1346
1347         if (cap->flags)
1348                 return -EINVAL;
1349
1350         switch (cap->cap) {
1351         case KVM_CAP_S390_CSS_SUPPORT:
1352                 if (!vcpu->kvm->arch.css_support) {
1353                         vcpu->kvm->arch.css_support = 1;
1354                         trace_kvm_s390_enable_css(vcpu->kvm);
1355                 }
1356                 r = 0;
1357                 break;
1358         default:
1359                 r = -EINVAL;
1360                 break;
1361         }
1362         return r;
1363 }
1364
1365 long kvm_arch_vcpu_ioctl(struct file *filp,
1366                          unsigned int ioctl, unsigned long arg)
1367 {
1368         struct kvm_vcpu *vcpu = filp->private_data;
1369         void __user *argp = (void __user *)arg;
1370         int idx;
1371         long r;
1372
1373         switch (ioctl) {
1374         case KVM_S390_INTERRUPT: {
1375                 struct kvm_s390_interrupt s390int;
1376
1377                 r = -EFAULT;
1378                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1379                         break;
1380                 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1381                 break;
1382         }
1383         case KVM_S390_STORE_STATUS:
1384                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1385                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1386                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1387                 break;
1388         case KVM_S390_SET_INITIAL_PSW: {
1389                 psw_t psw;
1390
1391                 r = -EFAULT;
1392                 if (copy_from_user(&psw, argp, sizeof(psw)))
1393                         break;
1394                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1395                 break;
1396         }
1397         case KVM_S390_INITIAL_RESET:
1398                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1399                 break;
1400         case KVM_SET_ONE_REG:
1401         case KVM_GET_ONE_REG: {
1402                 struct kvm_one_reg reg;
1403                 r = -EFAULT;
1404                 if (copy_from_user(&reg, argp, sizeof(reg)))
1405                         break;
1406                 if (ioctl == KVM_SET_ONE_REG)
1407                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1408                 else
1409                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1410                 break;
1411         }
1412 #ifdef CONFIG_KVM_S390_UCONTROL
1413         case KVM_S390_UCAS_MAP: {
1414                 struct kvm_s390_ucas_mapping ucasmap;
1415
1416                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1417                         r = -EFAULT;
1418                         break;
1419                 }
1420
1421                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1422                         r = -EINVAL;
1423                         break;
1424                 }
1425
1426                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1427                                      ucasmap.vcpu_addr, ucasmap.length);
1428                 break;
1429         }
1430         case KVM_S390_UCAS_UNMAP: {
1431                 struct kvm_s390_ucas_mapping ucasmap;
1432
1433                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1434                         r = -EFAULT;
1435                         break;
1436                 }
1437
1438                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1439                         r = -EINVAL;
1440                         break;
1441                 }
1442
1443                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1444                         ucasmap.length);
1445                 break;
1446         }
1447 #endif
1448         case KVM_S390_VCPU_FAULT: {
1449                 r = gmap_fault(arg, vcpu->arch.gmap);
1450                 if (!IS_ERR_VALUE(r))
1451                         r = 0;
1452                 break;
1453         }
1454         case KVM_ENABLE_CAP:
1455         {
1456                 struct kvm_enable_cap cap;
1457                 r = -EFAULT;
1458                 if (copy_from_user(&cap, argp, sizeof(cap)))
1459                         break;
1460                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1461                 break;
1462         }
1463         default:
1464                 r = -ENOTTY;
1465         }
1466         return r;
1467 }
1468
1469 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1470 {
1471 #ifdef CONFIG_KVM_S390_UCONTROL
1472         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1473                  && (kvm_is_ucontrol(vcpu->kvm))) {
1474                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1475                 get_page(vmf->page);
1476                 return 0;
1477         }
1478 #endif
1479         return VM_FAULT_SIGBUS;
1480 }
1481
1482 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
1483                            struct kvm_memory_slot *dont)
1484 {
1485 }
1486
1487 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1488                             unsigned long npages)
1489 {
1490         return 0;
1491 }
1492
1493 void kvm_arch_memslots_updated(struct kvm *kvm)
1494 {
1495 }
1496
1497 /* Section: memory related */
1498 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1499                                    struct kvm_memory_slot *memslot,
1500                                    struct kvm_userspace_memory_region *mem,
1501                                    enum kvm_mr_change change)
1502 {
1503         /* A few sanity checks. We can have memory slots which have to be
1504            located/ended at a segment boundary (1MB). The memory in userland is
1505            ok to be fragmented into various different vmas. It is okay to mmap()
1506            and munmap() stuff in this slot after doing this call at any time */
1507
1508         if (mem->userspace_addr & 0xffffful)
1509                 return -EINVAL;
1510
1511         if (mem->memory_size & 0xffffful)
1512                 return -EINVAL;
1513
1514         return 0;
1515 }
1516
1517 void kvm_arch_commit_memory_region(struct kvm *kvm,
1518                                 struct kvm_userspace_memory_region *mem,
1519                                 const struct kvm_memory_slot *old,
1520                                 enum kvm_mr_change change)
1521 {
1522         int rc;
1523
1524         /* If the basics of the memslot do not change, we do not want
1525          * to update the gmap. Every update causes several unnecessary
1526          * segment translation exceptions. This is usually handled just
1527          * fine by the normal fault handler + gmap, but it will also
1528          * cause faults on the prefix page of running guest CPUs.
1529          */
1530         if (old->userspace_addr == mem->userspace_addr &&
1531             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1532             old->npages * PAGE_SIZE == mem->memory_size)
1533                 return;
1534
1535         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1536                 mem->guest_phys_addr, mem->memory_size);
1537         if (rc)
1538                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1539         return;
1540 }
1541
1542 void kvm_arch_flush_shadow_all(struct kvm *kvm)
1543 {
1544 }
1545
1546 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1547                                    struct kvm_memory_slot *slot)
1548 {
1549 }
1550
1551 static int __init kvm_s390_init(void)
1552 {
1553         int ret;
1554         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1555         if (ret)
1556                 return ret;
1557
1558         /*
1559          * guests can ask for up to 255+1 double words, we need a full page
1560          * to hold the maximum amount of facilities. On the other hand, we
1561          * only set facilities that are known to work in KVM.
1562          */
1563         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1564         if (!vfacilities) {
1565                 kvm_exit();
1566                 return -ENOMEM;
1567         }
1568         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1569         vfacilities[0] &= 0xff82fff3f4fc2000UL;
1570         vfacilities[1] &= 0x005c000000000000UL;
1571         return 0;
1572 }
1573
1574 static void __exit kvm_s390_exit(void)
1575 {
1576         free_page((unsigned long) vfacilities);
1577         kvm_exit();
1578 }
1579
1580 module_init(kvm_s390_init);
1581 module_exit(kvm_s390_exit);
1582
1583 /*
1584  * Enable autoloading of the kvm module.
1585  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1586  * since x86 takes a different approach.
1587  */
1588 #include <linux/miscdevice.h>
1589 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1590 MODULE_ALIAS("devname:kvm");