2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <asm/asm-offsets.h>
30 #include <asm/lowcore.h>
31 #include <asm/pgtable.h>
33 #include <asm/switch_to.h>
38 #define CREATE_TRACE_POINTS
40 #include "trace-s390.h"
42 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
44 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
46 struct kvm_stats_debugfs_item debugfs_entries[] = {
47 { "userspace_handled", VCPU_STAT(exit_userspace) },
48 { "exit_null", VCPU_STAT(exit_null) },
49 { "exit_validity", VCPU_STAT(exit_validity) },
50 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
51 { "exit_external_request", VCPU_STAT(exit_external_request) },
52 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
53 { "exit_instruction", VCPU_STAT(exit_instruction) },
54 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
55 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
56 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
57 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
58 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
59 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
60 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
61 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
62 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
63 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
64 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
65 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
66 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
67 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
68 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
69 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
70 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
71 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
72 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
73 { "instruction_spx", VCPU_STAT(instruction_spx) },
74 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
75 { "instruction_stap", VCPU_STAT(instruction_stap) },
76 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
77 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
78 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
79 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
80 { "instruction_essa", VCPU_STAT(instruction_essa) },
81 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
82 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
83 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
84 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
85 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
86 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
87 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
88 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
89 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
90 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
91 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
92 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
93 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
94 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
95 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
96 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
97 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
98 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
99 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
100 { "diagnose_10", VCPU_STAT(diagnose_10) },
101 { "diagnose_44", VCPU_STAT(diagnose_44) },
102 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
106 /* upper facilities limit for kvm */
107 unsigned long kvm_s390_fac_list_mask[] = {
108 0xff82fffbf4fc2000UL,
109 0x005c000000000000UL,
112 unsigned long kvm_s390_fac_list_mask_size(void)
114 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
115 return ARRAY_SIZE(kvm_s390_fac_list_mask);
118 static struct gmap_notifier gmap_notifier;
120 /* Section: not file related */
121 int kvm_arch_hardware_enable(void)
123 /* every s390 is virtualization enabled ;-) */
127 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
129 int kvm_arch_hardware_setup(void)
131 gmap_notifier.notifier_call = kvm_gmap_notifier;
132 gmap_register_ipte_notifier(&gmap_notifier);
136 void kvm_arch_hardware_unsetup(void)
138 gmap_unregister_ipte_notifier(&gmap_notifier);
141 int kvm_arch_init(void *opaque)
143 /* Register floating interrupt controller interface. */
144 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
147 /* Section: device related */
148 long kvm_arch_dev_ioctl(struct file *filp,
149 unsigned int ioctl, unsigned long arg)
151 if (ioctl == KVM_S390_ENABLE_SIE)
152 return s390_enable_sie();
156 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
161 case KVM_CAP_S390_PSW:
162 case KVM_CAP_S390_GMAP:
163 case KVM_CAP_SYNC_MMU:
164 #ifdef CONFIG_KVM_S390_UCONTROL
165 case KVM_CAP_S390_UCONTROL:
167 case KVM_CAP_ASYNC_PF:
168 case KVM_CAP_SYNC_REGS:
169 case KVM_CAP_ONE_REG:
170 case KVM_CAP_ENABLE_CAP:
171 case KVM_CAP_S390_CSS_SUPPORT:
173 case KVM_CAP_IOEVENTFD:
174 case KVM_CAP_DEVICE_CTRL:
175 case KVM_CAP_ENABLE_CAP_VM:
176 case KVM_CAP_S390_IRQCHIP:
177 case KVM_CAP_VM_ATTRIBUTES:
178 case KVM_CAP_MP_STATE:
179 case KVM_CAP_S390_USER_SIGP:
180 case KVM_CAP_S390_USER_STSI:
181 case KVM_CAP_S390_SKEYS:
184 case KVM_CAP_S390_MEM_OP:
187 case KVM_CAP_NR_VCPUS:
188 case KVM_CAP_MAX_VCPUS:
191 case KVM_CAP_NR_MEMSLOTS:
192 r = KVM_USER_MEM_SLOTS;
194 case KVM_CAP_S390_COW:
195 r = MACHINE_HAS_ESOP;
197 case KVM_CAP_S390_VECTOR_REGISTERS:
206 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
207 struct kvm_memory_slot *memslot)
209 gfn_t cur_gfn, last_gfn;
210 unsigned long address;
211 struct gmap *gmap = kvm->arch.gmap;
213 down_read(&gmap->mm->mmap_sem);
214 /* Loop over all guest pages */
215 last_gfn = memslot->base_gfn + memslot->npages;
216 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
217 address = gfn_to_hva_memslot(memslot, cur_gfn);
219 if (gmap_test_and_clear_dirty(address, gmap))
220 mark_page_dirty(kvm, cur_gfn);
222 up_read(&gmap->mm->mmap_sem);
225 /* Section: vm related */
227 * Get (and clear) the dirty memory log for a memory slot.
229 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
230 struct kvm_dirty_log *log)
234 struct kvm_memory_slot *memslot;
237 mutex_lock(&kvm->slots_lock);
240 if (log->slot >= KVM_USER_MEM_SLOTS)
243 memslot = id_to_memslot(kvm->memslots, log->slot);
245 if (!memslot->dirty_bitmap)
248 kvm_s390_sync_dirty_log(kvm, memslot);
249 r = kvm_get_dirty_log(kvm, log, &is_dirty);
253 /* Clear the dirty log */
255 n = kvm_dirty_bitmap_bytes(memslot);
256 memset(memslot->dirty_bitmap, 0, n);
260 mutex_unlock(&kvm->slots_lock);
264 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
272 case KVM_CAP_S390_IRQCHIP:
273 kvm->arch.use_irqchip = 1;
276 case KVM_CAP_S390_USER_SIGP:
277 kvm->arch.user_sigp = 1;
280 case KVM_CAP_S390_VECTOR_REGISTERS:
281 if (MACHINE_HAS_VX) {
282 set_kvm_facility(kvm->arch.model.fac->mask, 129);
283 set_kvm_facility(kvm->arch.model.fac->list, 129);
288 case KVM_CAP_S390_USER_STSI:
289 kvm->arch.user_stsi = 1;
299 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
303 switch (attr->attr) {
304 case KVM_S390_VM_MEM_LIMIT_SIZE:
306 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
316 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
320 switch (attr->attr) {
321 case KVM_S390_VM_MEM_ENABLE_CMMA:
323 mutex_lock(&kvm->lock);
324 if (atomic_read(&kvm->online_vcpus) == 0) {
325 kvm->arch.use_cmma = 1;
328 mutex_unlock(&kvm->lock);
330 case KVM_S390_VM_MEM_CLR_CMMA:
331 mutex_lock(&kvm->lock);
332 idx = srcu_read_lock(&kvm->srcu);
333 s390_reset_cmma(kvm->arch.gmap->mm);
334 srcu_read_unlock(&kvm->srcu, idx);
335 mutex_unlock(&kvm->lock);
338 case KVM_S390_VM_MEM_LIMIT_SIZE: {
339 unsigned long new_limit;
341 if (kvm_is_ucontrol(kvm))
344 if (get_user(new_limit, (u64 __user *)attr->addr))
347 if (new_limit > kvm->arch.gmap->asce_end)
351 mutex_lock(&kvm->lock);
352 if (atomic_read(&kvm->online_vcpus) == 0) {
353 /* gmap_alloc will round the limit up */
354 struct gmap *new = gmap_alloc(current->mm, new_limit);
359 gmap_free(kvm->arch.gmap);
361 kvm->arch.gmap = new;
365 mutex_unlock(&kvm->lock);
375 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
377 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
379 struct kvm_vcpu *vcpu;
382 if (!test_kvm_facility(kvm, 76))
385 mutex_lock(&kvm->lock);
386 switch (attr->attr) {
387 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
389 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
390 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
391 kvm->arch.crypto.aes_kw = 1;
393 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
395 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
396 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
397 kvm->arch.crypto.dea_kw = 1;
399 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
400 kvm->arch.crypto.aes_kw = 0;
401 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
402 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
404 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
405 kvm->arch.crypto.dea_kw = 0;
406 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
407 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
410 mutex_unlock(&kvm->lock);
414 kvm_for_each_vcpu(i, vcpu, kvm) {
415 kvm_s390_vcpu_crypto_setup(vcpu);
418 mutex_unlock(&kvm->lock);
422 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
426 if (copy_from_user(>od_high, (void __user *)attr->addr,
436 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
438 struct kvm_vcpu *cur_vcpu;
439 unsigned int vcpu_idx;
443 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
446 r = store_tod_clock(&host_tod);
450 mutex_lock(&kvm->lock);
451 kvm->arch.epoch = gtod - host_tod;
452 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
453 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
456 mutex_unlock(&kvm->lock);
460 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
467 switch (attr->attr) {
468 case KVM_S390_VM_TOD_HIGH:
469 ret = kvm_s390_set_tod_high(kvm, attr);
471 case KVM_S390_VM_TOD_LOW:
472 ret = kvm_s390_set_tod_low(kvm, attr);
481 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
485 if (copy_to_user((void __user *)attr->addr, >od_high,
492 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
497 r = store_tod_clock(&host_tod);
501 gtod = host_tod + kvm->arch.epoch;
502 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
508 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
515 switch (attr->attr) {
516 case KVM_S390_VM_TOD_HIGH:
517 ret = kvm_s390_get_tod_high(kvm, attr);
519 case KVM_S390_VM_TOD_LOW:
520 ret = kvm_s390_get_tod_low(kvm, attr);
529 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
531 struct kvm_s390_vm_cpu_processor *proc;
534 mutex_lock(&kvm->lock);
535 if (atomic_read(&kvm->online_vcpus)) {
539 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
544 if (!copy_from_user(proc, (void __user *)attr->addr,
546 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
547 sizeof(struct cpuid));
548 kvm->arch.model.ibc = proc->ibc;
549 memcpy(kvm->arch.model.fac->list, proc->fac_list,
550 S390_ARCH_FAC_LIST_SIZE_BYTE);
555 mutex_unlock(&kvm->lock);
559 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
563 switch (attr->attr) {
564 case KVM_S390_VM_CPU_PROCESSOR:
565 ret = kvm_s390_set_processor(kvm, attr);
571 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
573 struct kvm_s390_vm_cpu_processor *proc;
576 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
581 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
582 proc->ibc = kvm->arch.model.ibc;
583 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
584 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
591 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
593 struct kvm_s390_vm_cpu_machine *mach;
596 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
601 get_cpu_id((struct cpuid *) &mach->cpuid);
602 mach->ibc = sclp_get_ibc();
603 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
604 S390_ARCH_FAC_LIST_SIZE_BYTE);
605 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
606 S390_ARCH_FAC_LIST_SIZE_BYTE);
607 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
614 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
618 switch (attr->attr) {
619 case KVM_S390_VM_CPU_PROCESSOR:
620 ret = kvm_s390_get_processor(kvm, attr);
622 case KVM_S390_VM_CPU_MACHINE:
623 ret = kvm_s390_get_machine(kvm, attr);
629 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
633 switch (attr->group) {
634 case KVM_S390_VM_MEM_CTRL:
635 ret = kvm_s390_set_mem_control(kvm, attr);
637 case KVM_S390_VM_TOD:
638 ret = kvm_s390_set_tod(kvm, attr);
640 case KVM_S390_VM_CPU_MODEL:
641 ret = kvm_s390_set_cpu_model(kvm, attr);
643 case KVM_S390_VM_CRYPTO:
644 ret = kvm_s390_vm_set_crypto(kvm, attr);
654 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
658 switch (attr->group) {
659 case KVM_S390_VM_MEM_CTRL:
660 ret = kvm_s390_get_mem_control(kvm, attr);
662 case KVM_S390_VM_TOD:
663 ret = kvm_s390_get_tod(kvm, attr);
665 case KVM_S390_VM_CPU_MODEL:
666 ret = kvm_s390_get_cpu_model(kvm, attr);
676 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
680 switch (attr->group) {
681 case KVM_S390_VM_MEM_CTRL:
682 switch (attr->attr) {
683 case KVM_S390_VM_MEM_ENABLE_CMMA:
684 case KVM_S390_VM_MEM_CLR_CMMA:
685 case KVM_S390_VM_MEM_LIMIT_SIZE:
693 case KVM_S390_VM_TOD:
694 switch (attr->attr) {
695 case KVM_S390_VM_TOD_LOW:
696 case KVM_S390_VM_TOD_HIGH:
704 case KVM_S390_VM_CPU_MODEL:
705 switch (attr->attr) {
706 case KVM_S390_VM_CPU_PROCESSOR:
707 case KVM_S390_VM_CPU_MACHINE:
715 case KVM_S390_VM_CRYPTO:
716 switch (attr->attr) {
717 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
718 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
719 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
720 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
736 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
740 unsigned long curkey;
743 if (args->flags != 0)
746 /* Is this guest using storage keys? */
747 if (!mm_use_skey(current->mm))
748 return KVM_S390_GET_SKEYS_NONE;
750 /* Enforce sane limit on memory allocation */
751 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
754 keys = kmalloc_array(args->count, sizeof(uint8_t),
755 GFP_KERNEL | __GFP_NOWARN);
757 keys = vmalloc(sizeof(uint8_t) * args->count);
761 for (i = 0; i < args->count; i++) {
762 hva = gfn_to_hva(kvm, args->start_gfn + i);
763 if (kvm_is_error_hva(hva)) {
768 curkey = get_guest_storage_key(current->mm, hva);
769 if (IS_ERR_VALUE(curkey)) {
776 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
777 sizeof(uint8_t) * args->count);
785 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
791 if (args->flags != 0)
794 /* Enforce sane limit on memory allocation */
795 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
798 keys = kmalloc_array(args->count, sizeof(uint8_t),
799 GFP_KERNEL | __GFP_NOWARN);
801 keys = vmalloc(sizeof(uint8_t) * args->count);
805 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
806 sizeof(uint8_t) * args->count);
812 /* Enable storage key handling for the guest */
815 for (i = 0; i < args->count; i++) {
816 hva = gfn_to_hva(kvm, args->start_gfn + i);
817 if (kvm_is_error_hva(hva)) {
822 /* Lowest order bit is reserved */
823 if (keys[i] & 0x01) {
828 r = set_guest_storage_key(current->mm, hva,
829 (unsigned long)keys[i], 0);
838 long kvm_arch_vm_ioctl(struct file *filp,
839 unsigned int ioctl, unsigned long arg)
841 struct kvm *kvm = filp->private_data;
842 void __user *argp = (void __user *)arg;
843 struct kvm_device_attr attr;
847 case KVM_S390_INTERRUPT: {
848 struct kvm_s390_interrupt s390int;
851 if (copy_from_user(&s390int, argp, sizeof(s390int)))
853 r = kvm_s390_inject_vm(kvm, &s390int);
856 case KVM_ENABLE_CAP: {
857 struct kvm_enable_cap cap;
859 if (copy_from_user(&cap, argp, sizeof(cap)))
861 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
864 case KVM_CREATE_IRQCHIP: {
865 struct kvm_irq_routing_entry routing;
868 if (kvm->arch.use_irqchip) {
869 /* Set up dummy routing. */
870 memset(&routing, 0, sizeof(routing));
871 kvm_set_irq_routing(kvm, &routing, 0, 0);
876 case KVM_SET_DEVICE_ATTR: {
878 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
880 r = kvm_s390_vm_set_attr(kvm, &attr);
883 case KVM_GET_DEVICE_ATTR: {
885 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
887 r = kvm_s390_vm_get_attr(kvm, &attr);
890 case KVM_HAS_DEVICE_ATTR: {
892 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
894 r = kvm_s390_vm_has_attr(kvm, &attr);
897 case KVM_S390_GET_SKEYS: {
898 struct kvm_s390_skeys args;
901 if (copy_from_user(&args, argp,
902 sizeof(struct kvm_s390_skeys)))
904 r = kvm_s390_get_skeys(kvm, &args);
907 case KVM_S390_SET_SKEYS: {
908 struct kvm_s390_skeys args;
911 if (copy_from_user(&args, argp,
912 sizeof(struct kvm_s390_skeys)))
914 r = kvm_s390_set_skeys(kvm, &args);
924 static int kvm_s390_query_ap_config(u8 *config)
926 u32 fcn_code = 0x04000000UL;
929 memset(config, 0, 128);
933 ".long 0xb2af0000\n" /* PQAP(QCI) */
939 : "r" (fcn_code), "r" (config)
940 : "cc", "0", "2", "memory"
946 static int kvm_s390_apxa_installed(void)
951 if (test_facility(2) && test_facility(12)) {
952 cc = kvm_s390_query_ap_config(config);
955 pr_err("PQAP(QCI) failed with cc=%d", cc);
957 return config[0] & 0x40;
963 static void kvm_s390_set_crycb_format(struct kvm *kvm)
965 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
967 if (kvm_s390_apxa_installed())
968 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
970 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
973 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
976 cpu_id->version = 0xff;
979 static int kvm_s390_crypto_init(struct kvm *kvm)
981 if (!test_kvm_facility(kvm, 76))
984 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
985 GFP_KERNEL | GFP_DMA);
986 if (!kvm->arch.crypto.crycb)
989 kvm_s390_set_crycb_format(kvm);
991 /* Enable AES/DEA protected key functions by default */
992 kvm->arch.crypto.aes_kw = 1;
993 kvm->arch.crypto.dea_kw = 1;
994 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
995 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
996 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
997 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1002 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1005 char debug_name[16];
1006 static unsigned long sca_offset;
1009 #ifdef CONFIG_KVM_S390_UCONTROL
1010 if (type & ~KVM_VM_S390_UCONTROL)
1012 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1019 rc = s390_enable_sie();
1025 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1028 spin_lock(&kvm_lock);
1029 sca_offset = (sca_offset + 16) & 0x7f0;
1030 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1031 spin_unlock(&kvm_lock);
1033 sprintf(debug_name, "kvm-%u", current->pid);
1035 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
1040 * The architectural maximum amount of facilities is 16 kbit. To store
1041 * this amount, 2 kbyte of memory is required. Thus we need a full
1042 * page to hold the guest facility list (arch.model.fac->list) and the
1043 * facility mask (arch.model.fac->mask). Its address size has to be
1044 * 31 bits and word aligned.
1046 kvm->arch.model.fac =
1047 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1048 if (!kvm->arch.model.fac)
1051 /* Populate the facility mask initially. */
1052 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1053 S390_ARCH_FAC_LIST_SIZE_BYTE);
1054 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1055 if (i < kvm_s390_fac_list_mask_size())
1056 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1058 kvm->arch.model.fac->mask[i] = 0UL;
1061 /* Populate the facility list initially. */
1062 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1063 S390_ARCH_FAC_LIST_SIZE_BYTE);
1065 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1066 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
1068 if (kvm_s390_crypto_init(kvm) < 0)
1071 spin_lock_init(&kvm->arch.float_int.lock);
1072 INIT_LIST_HEAD(&kvm->arch.float_int.list);
1073 init_waitqueue_head(&kvm->arch.ipte_wq);
1074 mutex_init(&kvm->arch.ipte_mutex);
1076 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1077 VM_EVENT(kvm, 3, "%s", "vm created");
1079 if (type & KVM_VM_S390_UCONTROL) {
1080 kvm->arch.gmap = NULL;
1082 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1083 if (!kvm->arch.gmap)
1085 kvm->arch.gmap->private = kvm;
1086 kvm->arch.gmap->pfault_enabled = 0;
1089 kvm->arch.css_support = 0;
1090 kvm->arch.use_irqchip = 0;
1091 kvm->arch.epoch = 0;
1093 spin_lock_init(&kvm->arch.start_stop_lock);
1097 kfree(kvm->arch.crypto.crycb);
1098 free_page((unsigned long)kvm->arch.model.fac);
1099 debug_unregister(kvm->arch.dbf);
1100 free_page((unsigned long)(kvm->arch.sca));
1104 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1106 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1107 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1108 kvm_s390_clear_local_irqs(vcpu);
1109 kvm_clear_async_pf_completion_queue(vcpu);
1110 if (!kvm_is_ucontrol(vcpu->kvm)) {
1111 clear_bit(63 - vcpu->vcpu_id,
1112 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1113 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1114 (__u64) vcpu->arch.sie_block)
1115 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1119 if (kvm_is_ucontrol(vcpu->kvm))
1120 gmap_free(vcpu->arch.gmap);
1122 if (kvm_s390_cmma_enabled(vcpu->kvm))
1123 kvm_s390_vcpu_unsetup_cmma(vcpu);
1124 free_page((unsigned long)(vcpu->arch.sie_block));
1126 kvm_vcpu_uninit(vcpu);
1127 kmem_cache_free(kvm_vcpu_cache, vcpu);
1130 static void kvm_free_vcpus(struct kvm *kvm)
1133 struct kvm_vcpu *vcpu;
1135 kvm_for_each_vcpu(i, vcpu, kvm)
1136 kvm_arch_vcpu_destroy(vcpu);
1138 mutex_lock(&kvm->lock);
1139 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1140 kvm->vcpus[i] = NULL;
1142 atomic_set(&kvm->online_vcpus, 0);
1143 mutex_unlock(&kvm->lock);
1146 void kvm_arch_destroy_vm(struct kvm *kvm)
1148 kvm_free_vcpus(kvm);
1149 free_page((unsigned long)kvm->arch.model.fac);
1150 free_page((unsigned long)(kvm->arch.sca));
1151 debug_unregister(kvm->arch.dbf);
1152 kfree(kvm->arch.crypto.crycb);
1153 if (!kvm_is_ucontrol(kvm))
1154 gmap_free(kvm->arch.gmap);
1155 kvm_s390_destroy_adapters(kvm);
1156 kvm_s390_clear_float_irqs(kvm);
1159 /* Section: vcpu related */
1160 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1162 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1163 if (!vcpu->arch.gmap)
1165 vcpu->arch.gmap->private = vcpu->kvm;
1170 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1172 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1173 kvm_clear_async_pf_completion_queue(vcpu);
1174 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1180 if (test_kvm_facility(vcpu->kvm, 129))
1181 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1183 if (kvm_is_ucontrol(vcpu->kvm))
1184 return __kvm_ucontrol_vcpu_init(vcpu);
1189 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1191 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1192 if (test_kvm_facility(vcpu->kvm, 129))
1193 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1195 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1196 save_access_regs(vcpu->arch.host_acrs);
1197 if (test_kvm_facility(vcpu->kvm, 129)) {
1198 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1199 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1201 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1202 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1204 restore_access_regs(vcpu->run->s.regs.acrs);
1205 gmap_enable(vcpu->arch.gmap);
1206 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1209 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1211 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1212 gmap_disable(vcpu->arch.gmap);
1213 if (test_kvm_facility(vcpu->kvm, 129)) {
1214 save_fp_ctl(&vcpu->run->s.regs.fpc);
1215 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1217 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1218 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1220 save_access_regs(vcpu->run->s.regs.acrs);
1221 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1222 if (test_kvm_facility(vcpu->kvm, 129))
1223 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1225 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1226 restore_access_regs(vcpu->arch.host_acrs);
1229 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1231 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1232 vcpu->arch.sie_block->gpsw.mask = 0UL;
1233 vcpu->arch.sie_block->gpsw.addr = 0UL;
1234 kvm_s390_set_prefix(vcpu, 0);
1235 vcpu->arch.sie_block->cputm = 0UL;
1236 vcpu->arch.sie_block->ckc = 0UL;
1237 vcpu->arch.sie_block->todpr = 0;
1238 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1239 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1240 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1241 vcpu->arch.guest_fpregs.fpc = 0;
1242 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1243 vcpu->arch.sie_block->gbea = 1;
1244 vcpu->arch.sie_block->pp = 0;
1245 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1246 kvm_clear_async_pf_completion_queue(vcpu);
1247 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1248 kvm_s390_vcpu_stop(vcpu);
1249 kvm_s390_clear_local_irqs(vcpu);
1252 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1254 mutex_lock(&vcpu->kvm->lock);
1255 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1256 mutex_unlock(&vcpu->kvm->lock);
1257 if (!kvm_is_ucontrol(vcpu->kvm))
1258 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1261 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1263 if (!test_kvm_facility(vcpu->kvm, 76))
1266 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1268 if (vcpu->kvm->arch.crypto.aes_kw)
1269 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1270 if (vcpu->kvm->arch.crypto.dea_kw)
1271 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1273 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1276 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1278 free_page(vcpu->arch.sie_block->cbrlo);
1279 vcpu->arch.sie_block->cbrlo = 0;
1282 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1284 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1285 if (!vcpu->arch.sie_block->cbrlo)
1288 vcpu->arch.sie_block->ecb2 |= 0x80;
1289 vcpu->arch.sie_block->ecb2 &= ~0x08;
1293 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1295 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1297 vcpu->arch.cpu_id = model->cpu_id;
1298 vcpu->arch.sie_block->ibc = model->ibc;
1299 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1302 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1306 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1310 kvm_s390_vcpu_setup_model(vcpu);
1312 vcpu->arch.sie_block->ecb = 6;
1313 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1314 vcpu->arch.sie_block->ecb |= 0x10;
1316 vcpu->arch.sie_block->ecb2 = 8;
1317 vcpu->arch.sie_block->eca = 0xC1002000U;
1318 if (sclp_has_siif())
1319 vcpu->arch.sie_block->eca |= 1;
1320 if (sclp_has_sigpif())
1321 vcpu->arch.sie_block->eca |= 0x10000000U;
1322 if (test_kvm_facility(vcpu->kvm, 129)) {
1323 vcpu->arch.sie_block->eca |= 0x00020000;
1324 vcpu->arch.sie_block->ecd |= 0x20000000;
1326 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1328 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1329 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1333 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1334 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1336 kvm_s390_vcpu_crypto_setup(vcpu);
1341 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1344 struct kvm_vcpu *vcpu;
1345 struct sie_page *sie_page;
1348 if (id >= KVM_MAX_VCPUS)
1353 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1357 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1361 vcpu->arch.sie_block = &sie_page->sie_block;
1362 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1363 vcpu->arch.host_vregs = &sie_page->vregs;
1365 vcpu->arch.sie_block->icpua = id;
1366 if (!kvm_is_ucontrol(kvm)) {
1367 if (!kvm->arch.sca) {
1371 if (!kvm->arch.sca->cpu[id].sda)
1372 kvm->arch.sca->cpu[id].sda =
1373 (__u64) vcpu->arch.sie_block;
1374 vcpu->arch.sie_block->scaoh =
1375 (__u32)(((__u64)kvm->arch.sca) >> 32);
1376 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1377 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1380 spin_lock_init(&vcpu->arch.local_int.lock);
1381 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1382 vcpu->arch.local_int.wq = &vcpu->wq;
1383 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1385 rc = kvm_vcpu_init(vcpu, kvm, id);
1387 goto out_free_sie_block;
1388 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1389 vcpu->arch.sie_block);
1390 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1394 free_page((unsigned long)(vcpu->arch.sie_block));
1396 kmem_cache_free(kvm_vcpu_cache, vcpu);
1401 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1403 return kvm_s390_vcpu_has_irq(vcpu, 0);
1406 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1408 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1411 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1413 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1417 * Kick a guest cpu out of SIE and wait until SIE is not running.
1418 * If the CPU is not running (e.g. waiting as idle) the function will
1419 * return immediately. */
1420 void exit_sie(struct kvm_vcpu *vcpu)
1422 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1423 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1427 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1428 void exit_sie_sync(struct kvm_vcpu *vcpu)
1430 s390_vcpu_block(vcpu);
1434 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1437 struct kvm *kvm = gmap->private;
1438 struct kvm_vcpu *vcpu;
1440 kvm_for_each_vcpu(i, vcpu, kvm) {
1441 /* match against both prefix pages */
1442 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1443 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1444 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1445 exit_sie_sync(vcpu);
1450 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1452 /* kvm common code refers to this, but never calls it */
1457 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1458 struct kvm_one_reg *reg)
1463 case KVM_REG_S390_TODPR:
1464 r = put_user(vcpu->arch.sie_block->todpr,
1465 (u32 __user *)reg->addr);
1467 case KVM_REG_S390_EPOCHDIFF:
1468 r = put_user(vcpu->arch.sie_block->epoch,
1469 (u64 __user *)reg->addr);
1471 case KVM_REG_S390_CPU_TIMER:
1472 r = put_user(vcpu->arch.sie_block->cputm,
1473 (u64 __user *)reg->addr);
1475 case KVM_REG_S390_CLOCK_COMP:
1476 r = put_user(vcpu->arch.sie_block->ckc,
1477 (u64 __user *)reg->addr);
1479 case KVM_REG_S390_PFTOKEN:
1480 r = put_user(vcpu->arch.pfault_token,
1481 (u64 __user *)reg->addr);
1483 case KVM_REG_S390_PFCOMPARE:
1484 r = put_user(vcpu->arch.pfault_compare,
1485 (u64 __user *)reg->addr);
1487 case KVM_REG_S390_PFSELECT:
1488 r = put_user(vcpu->arch.pfault_select,
1489 (u64 __user *)reg->addr);
1491 case KVM_REG_S390_PP:
1492 r = put_user(vcpu->arch.sie_block->pp,
1493 (u64 __user *)reg->addr);
1495 case KVM_REG_S390_GBEA:
1496 r = put_user(vcpu->arch.sie_block->gbea,
1497 (u64 __user *)reg->addr);
1506 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1507 struct kvm_one_reg *reg)
1512 case KVM_REG_S390_TODPR:
1513 r = get_user(vcpu->arch.sie_block->todpr,
1514 (u32 __user *)reg->addr);
1516 case KVM_REG_S390_EPOCHDIFF:
1517 r = get_user(vcpu->arch.sie_block->epoch,
1518 (u64 __user *)reg->addr);
1520 case KVM_REG_S390_CPU_TIMER:
1521 r = get_user(vcpu->arch.sie_block->cputm,
1522 (u64 __user *)reg->addr);
1524 case KVM_REG_S390_CLOCK_COMP:
1525 r = get_user(vcpu->arch.sie_block->ckc,
1526 (u64 __user *)reg->addr);
1528 case KVM_REG_S390_PFTOKEN:
1529 r = get_user(vcpu->arch.pfault_token,
1530 (u64 __user *)reg->addr);
1531 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1532 kvm_clear_async_pf_completion_queue(vcpu);
1534 case KVM_REG_S390_PFCOMPARE:
1535 r = get_user(vcpu->arch.pfault_compare,
1536 (u64 __user *)reg->addr);
1538 case KVM_REG_S390_PFSELECT:
1539 r = get_user(vcpu->arch.pfault_select,
1540 (u64 __user *)reg->addr);
1542 case KVM_REG_S390_PP:
1543 r = get_user(vcpu->arch.sie_block->pp,
1544 (u64 __user *)reg->addr);
1546 case KVM_REG_S390_GBEA:
1547 r = get_user(vcpu->arch.sie_block->gbea,
1548 (u64 __user *)reg->addr);
1557 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1559 kvm_s390_vcpu_initial_reset(vcpu);
1563 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1565 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
1569 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1571 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1575 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1576 struct kvm_sregs *sregs)
1578 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1579 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1580 restore_access_regs(vcpu->run->s.regs.acrs);
1584 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1585 struct kvm_sregs *sregs)
1587 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1588 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1592 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1594 if (test_fp_ctl(fpu->fpc))
1596 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1597 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1598 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1599 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1603 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1605 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1606 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1610 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1614 if (!is_vcpu_stopped(vcpu))
1617 vcpu->run->psw_mask = psw.mask;
1618 vcpu->run->psw_addr = psw.addr;
1623 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1624 struct kvm_translation *tr)
1626 return -EINVAL; /* not implemented yet */
1629 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1630 KVM_GUESTDBG_USE_HW_BP | \
1631 KVM_GUESTDBG_ENABLE)
1633 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1634 struct kvm_guest_debug *dbg)
1638 vcpu->guest_debug = 0;
1639 kvm_s390_clear_bp_data(vcpu);
1641 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1644 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1645 vcpu->guest_debug = dbg->control;
1646 /* enforce guest PER */
1647 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1649 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1650 rc = kvm_s390_import_bp_data(vcpu, dbg);
1652 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1653 vcpu->arch.guestdbg.last_bp = 0;
1657 vcpu->guest_debug = 0;
1658 kvm_s390_clear_bp_data(vcpu);
1659 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1665 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1666 struct kvm_mp_state *mp_state)
1668 /* CHECK_STOP and LOAD are not supported yet */
1669 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1670 KVM_MP_STATE_OPERATING;
1673 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1674 struct kvm_mp_state *mp_state)
1678 /* user space knows about this interface - let it control the state */
1679 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1681 switch (mp_state->mp_state) {
1682 case KVM_MP_STATE_STOPPED:
1683 kvm_s390_vcpu_stop(vcpu);
1685 case KVM_MP_STATE_OPERATING:
1686 kvm_s390_vcpu_start(vcpu);
1688 case KVM_MP_STATE_LOAD:
1689 case KVM_MP_STATE_CHECK_STOP:
1690 /* fall through - CHECK_STOP and LOAD are not supported yet */
1698 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1700 if (!MACHINE_IS_LPAR)
1702 /* only enable for z10 and later */
1703 if (!MACHINE_HAS_EDAT1)
1705 if (!kvm->arch.use_cmma)
1710 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1712 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1715 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1718 s390_vcpu_unblock(vcpu);
1720 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1721 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1722 * This ensures that the ipte instruction for this request has
1723 * already finished. We might race against a second unmapper that
1724 * wants to set the blocking bit. Lets just retry the request loop.
1726 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1728 rc = gmap_ipte_notify(vcpu->arch.gmap,
1729 kvm_s390_get_prefix(vcpu),
1736 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1737 vcpu->arch.sie_block->ihcpu = 0xffff;
1741 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1742 if (!ibs_enabled(vcpu)) {
1743 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1744 atomic_set_mask(CPUSTAT_IBS,
1745 &vcpu->arch.sie_block->cpuflags);
1750 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1751 if (ibs_enabled(vcpu)) {
1752 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1753 atomic_clear_mask(CPUSTAT_IBS,
1754 &vcpu->arch.sie_block->cpuflags);
1759 /* nothing to do, just clear the request */
1760 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1766 * kvm_arch_fault_in_page - fault-in guest page if necessary
1767 * @vcpu: The corresponding virtual cpu
1768 * @gpa: Guest physical address
1769 * @writable: Whether the page should be writable or not
1771 * Make sure that a guest page has been faulted-in on the host.
1773 * Return: Zero on success, negative error code otherwise.
1775 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1777 return gmap_fault(vcpu->arch.gmap, gpa,
1778 writable ? FAULT_FLAG_WRITE : 0);
1781 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1782 unsigned long token)
1784 struct kvm_s390_interrupt inti;
1785 struct kvm_s390_irq irq;
1788 irq.u.ext.ext_params2 = token;
1789 irq.type = KVM_S390_INT_PFAULT_INIT;
1790 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1792 inti.type = KVM_S390_INT_PFAULT_DONE;
1793 inti.parm64 = token;
1794 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1798 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1799 struct kvm_async_pf *work)
1801 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1802 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1805 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1806 struct kvm_async_pf *work)
1808 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1809 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1812 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1813 struct kvm_async_pf *work)
1815 /* s390 will always inject the page directly */
1818 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1821 * s390 will always inject the page directly,
1822 * but we still want check_async_completion to cleanup
1827 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1830 struct kvm_arch_async_pf arch;
1833 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1835 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1836 vcpu->arch.pfault_compare)
1838 if (psw_extint_disabled(vcpu))
1840 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1842 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1844 if (!vcpu->arch.gmap->pfault_enabled)
1847 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1848 hva += current->thread.gmap_addr & ~PAGE_MASK;
1849 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1852 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1856 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1861 * On s390 notifications for arriving pages will be delivered directly
1862 * to the guest but the house keeping for completed pfaults is
1863 * handled outside the worker.
1865 kvm_check_async_pf_completion(vcpu);
1867 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1872 if (test_cpu_flag(CIF_MCCK_PENDING))
1875 if (!kvm_is_ucontrol(vcpu->kvm)) {
1876 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1881 rc = kvm_s390_handle_requests(vcpu);
1885 if (guestdbg_enabled(vcpu)) {
1886 kvm_s390_backup_guest_per_regs(vcpu);
1887 kvm_s390_patch_guest_per_regs(vcpu);
1890 vcpu->arch.sie_block->icptcode = 0;
1891 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1892 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1893 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1898 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1900 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1904 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1905 trace_kvm_s390_sie_fault(vcpu);
1908 * We want to inject an addressing exception, which is defined as a
1909 * suppressing or terminating exception. However, since we came here
1910 * by a DAT access exception, the PSW still points to the faulting
1911 * instruction since DAT exceptions are nullifying. So we've got
1912 * to look up the current opcode to get the length of the instruction
1913 * to be able to forward the PSW.
1915 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1917 return kvm_s390_inject_prog_cond(vcpu, rc);
1918 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1920 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1923 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1927 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1928 vcpu->arch.sie_block->icptcode);
1929 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1931 if (guestdbg_enabled(vcpu))
1932 kvm_s390_restore_guest_per_regs(vcpu);
1934 if (exit_reason >= 0) {
1936 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1937 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1938 vcpu->run->s390_ucontrol.trans_exc_code =
1939 current->thread.gmap_addr;
1940 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1943 } else if (current->thread.gmap_pfault) {
1944 trace_kvm_s390_major_guest_pfault(vcpu);
1945 current->thread.gmap_pfault = 0;
1946 if (kvm_arch_setup_async_pf(vcpu)) {
1949 gpa_t gpa = current->thread.gmap_addr;
1950 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1955 rc = vcpu_post_run_fault_in_sie(vcpu);
1957 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1960 if (kvm_is_ucontrol(vcpu->kvm))
1961 /* Don't exit for host interrupts. */
1962 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1964 rc = kvm_handle_sie_intercept(vcpu);
1970 static int __vcpu_run(struct kvm_vcpu *vcpu)
1972 int rc, exit_reason;
1975 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1976 * ning the guest), so that memslots (and other stuff) are protected
1978 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1981 rc = vcpu_pre_run(vcpu);
1985 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1987 * As PF_VCPU will be used in fault handler, between
1988 * guest_enter and guest_exit should be no uaccess.
1993 exit_reason = sie64a(vcpu->arch.sie_block,
1994 vcpu->run->s.regs.gprs);
1996 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1998 rc = vcpu_post_run(vcpu, exit_reason);
1999 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2001 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2005 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2007 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2008 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2009 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2010 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2011 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2012 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2013 /* some control register changes require a tlb flush */
2014 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2016 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2017 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2018 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2019 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2020 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2021 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2023 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2024 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2025 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2026 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2027 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2028 kvm_clear_async_pf_completion_queue(vcpu);
2030 kvm_run->kvm_dirty_regs = 0;
2033 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2035 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2036 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2037 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2038 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2039 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2040 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2041 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2042 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2043 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2044 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2045 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2046 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2049 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2054 if (guestdbg_exit_pending(vcpu)) {
2055 kvm_s390_prepare_debug_exit(vcpu);
2059 if (vcpu->sigset_active)
2060 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2062 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2063 kvm_s390_vcpu_start(vcpu);
2064 } else if (is_vcpu_stopped(vcpu)) {
2065 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
2070 sync_regs(vcpu, kvm_run);
2073 rc = __vcpu_run(vcpu);
2075 if (signal_pending(current) && !rc) {
2076 kvm_run->exit_reason = KVM_EXIT_INTR;
2080 if (guestdbg_exit_pending(vcpu) && !rc) {
2081 kvm_s390_prepare_debug_exit(vcpu);
2085 if (rc == -EOPNOTSUPP) {
2086 /* intercept cannot be handled in-kernel, prepare kvm-run */
2087 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2088 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2089 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2090 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2094 if (rc == -EREMOTE) {
2095 /* intercept was handled, but userspace support is needed
2096 * kvm_run has been prepared by the handler */
2100 store_regs(vcpu, kvm_run);
2102 if (vcpu->sigset_active)
2103 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2105 vcpu->stat.exit_userspace++;
2110 * store status at address
2111 * we use have two special cases:
2112 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2113 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2115 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2117 unsigned char archmode = 1;
2122 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2123 if (write_guest_abs(vcpu, 163, &archmode, 1))
2125 gpa = SAVE_AREA_BASE;
2126 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2127 if (write_guest_real(vcpu, 163, &archmode, 1))
2129 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2131 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2132 vcpu->arch.guest_fpregs.fprs, 128);
2133 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2134 vcpu->run->s.regs.gprs, 128);
2135 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2136 &vcpu->arch.sie_block->gpsw, 16);
2137 px = kvm_s390_get_prefix(vcpu);
2138 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2140 rc |= write_guest_abs(vcpu,
2141 gpa + offsetof(struct save_area, fp_ctrl_reg),
2142 &vcpu->arch.guest_fpregs.fpc, 4);
2143 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2144 &vcpu->arch.sie_block->todpr, 4);
2145 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2146 &vcpu->arch.sie_block->cputm, 8);
2147 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2148 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2150 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2151 &vcpu->run->s.regs.acrs, 64);
2152 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2153 &vcpu->arch.sie_block->gcr, 128);
2154 return rc ? -EFAULT : 0;
2157 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2160 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2161 * copying in vcpu load/put. Lets update our copies before we save
2162 * it into the save area
2164 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2165 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2166 save_access_regs(vcpu->run->s.regs.acrs);
2168 return kvm_s390_store_status_unloaded(vcpu, addr);
2172 * store additional status at address
2174 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2177 /* Only bits 0-53 are used for address formation */
2178 if (!(gpa & ~0x3ff))
2181 return write_guest_abs(vcpu, gpa & ~0x3ff,
2182 (void *)&vcpu->run->s.regs.vrs, 512);
2185 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2187 if (!test_kvm_facility(vcpu->kvm, 129))
2191 * The guest VXRS are in the host VXRs due to the lazy
2192 * copying in vcpu load/put. Let's update our copies before we save
2193 * it into the save area.
2195 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2197 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2200 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2202 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2203 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
2204 exit_sie_sync(vcpu);
2207 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2210 struct kvm_vcpu *vcpu;
2212 kvm_for_each_vcpu(i, vcpu, kvm) {
2213 __disable_ibs_on_vcpu(vcpu);
2217 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2219 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2220 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2221 exit_sie_sync(vcpu);
2224 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2226 int i, online_vcpus, started_vcpus = 0;
2228 if (!is_vcpu_stopped(vcpu))
2231 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2232 /* Only one cpu at a time may enter/leave the STOPPED state. */
2233 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2234 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2236 for (i = 0; i < online_vcpus; i++) {
2237 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2241 if (started_vcpus == 0) {
2242 /* we're the only active VCPU -> speed it up */
2243 __enable_ibs_on_vcpu(vcpu);
2244 } else if (started_vcpus == 1) {
2246 * As we are starting a second VCPU, we have to disable
2247 * the IBS facility on all VCPUs to remove potentially
2248 * oustanding ENABLE requests.
2250 __disable_ibs_on_all_vcpus(vcpu->kvm);
2253 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2255 * Another VCPU might have used IBS while we were offline.
2256 * Let's play safe and flush the VCPU at startup.
2258 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2259 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2263 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2265 int i, online_vcpus, started_vcpus = 0;
2266 struct kvm_vcpu *started_vcpu = NULL;
2268 if (is_vcpu_stopped(vcpu))
2271 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2272 /* Only one cpu at a time may enter/leave the STOPPED state. */
2273 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2274 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2276 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2277 kvm_s390_clear_stop_irq(vcpu);
2279 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2280 __disable_ibs_on_vcpu(vcpu);
2282 for (i = 0; i < online_vcpus; i++) {
2283 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2285 started_vcpu = vcpu->kvm->vcpus[i];
2289 if (started_vcpus == 1) {
2291 * As we only have one VCPU left, we want to enable the
2292 * IBS facility for that VCPU to speed it up.
2294 __enable_ibs_on_vcpu(started_vcpu);
2297 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2301 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2302 struct kvm_enable_cap *cap)
2310 case KVM_CAP_S390_CSS_SUPPORT:
2311 if (!vcpu->kvm->arch.css_support) {
2312 vcpu->kvm->arch.css_support = 1;
2313 trace_kvm_s390_enable_css(vcpu->kvm);
2324 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2325 struct kvm_s390_mem_op *mop)
2327 void __user *uaddr = (void __user *)mop->buf;
2328 void *tmpbuf = NULL;
2330 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2331 | KVM_S390_MEMOP_F_CHECK_ONLY;
2333 if (mop->flags & ~supported_flags)
2336 if (mop->size > MEM_OP_MAX_SIZE)
2339 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2340 tmpbuf = vmalloc(mop->size);
2345 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2348 case KVM_S390_MEMOP_LOGICAL_READ:
2349 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2350 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2353 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2355 if (copy_to_user(uaddr, tmpbuf, mop->size))
2359 case KVM_S390_MEMOP_LOGICAL_WRITE:
2360 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2361 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2364 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2368 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2374 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2376 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2377 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2383 long kvm_arch_vcpu_ioctl(struct file *filp,
2384 unsigned int ioctl, unsigned long arg)
2386 struct kvm_vcpu *vcpu = filp->private_data;
2387 void __user *argp = (void __user *)arg;
2392 case KVM_S390_INTERRUPT: {
2393 struct kvm_s390_interrupt s390int;
2394 struct kvm_s390_irq s390irq;
2397 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2399 if (s390int_to_s390irq(&s390int, &s390irq))
2401 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2404 case KVM_S390_STORE_STATUS:
2405 idx = srcu_read_lock(&vcpu->kvm->srcu);
2406 r = kvm_s390_vcpu_store_status(vcpu, arg);
2407 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2409 case KVM_S390_SET_INITIAL_PSW: {
2413 if (copy_from_user(&psw, argp, sizeof(psw)))
2415 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2418 case KVM_S390_INITIAL_RESET:
2419 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2421 case KVM_SET_ONE_REG:
2422 case KVM_GET_ONE_REG: {
2423 struct kvm_one_reg reg;
2425 if (copy_from_user(®, argp, sizeof(reg)))
2427 if (ioctl == KVM_SET_ONE_REG)
2428 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
2430 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
2433 #ifdef CONFIG_KVM_S390_UCONTROL
2434 case KVM_S390_UCAS_MAP: {
2435 struct kvm_s390_ucas_mapping ucasmap;
2437 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2442 if (!kvm_is_ucontrol(vcpu->kvm)) {
2447 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2448 ucasmap.vcpu_addr, ucasmap.length);
2451 case KVM_S390_UCAS_UNMAP: {
2452 struct kvm_s390_ucas_mapping ucasmap;
2454 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2459 if (!kvm_is_ucontrol(vcpu->kvm)) {
2464 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2469 case KVM_S390_VCPU_FAULT: {
2470 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2473 case KVM_ENABLE_CAP:
2475 struct kvm_enable_cap cap;
2477 if (copy_from_user(&cap, argp, sizeof(cap)))
2479 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2482 case KVM_S390_MEM_OP: {
2483 struct kvm_s390_mem_op mem_op;
2485 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2486 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2497 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2499 #ifdef CONFIG_KVM_S390_UCONTROL
2500 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2501 && (kvm_is_ucontrol(vcpu->kvm))) {
2502 vmf->page = virt_to_page(vcpu->arch.sie_block);
2503 get_page(vmf->page);
2507 return VM_FAULT_SIGBUS;
2510 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2511 unsigned long npages)
2516 /* Section: memory related */
2517 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2518 struct kvm_memory_slot *memslot,
2519 struct kvm_userspace_memory_region *mem,
2520 enum kvm_mr_change change)
2522 /* A few sanity checks. We can have memory slots which have to be
2523 located/ended at a segment boundary (1MB). The memory in userland is
2524 ok to be fragmented into various different vmas. It is okay to mmap()
2525 and munmap() stuff in this slot after doing this call at any time */
2527 if (mem->userspace_addr & 0xffffful)
2530 if (mem->memory_size & 0xffffful)
2536 void kvm_arch_commit_memory_region(struct kvm *kvm,
2537 struct kvm_userspace_memory_region *mem,
2538 const struct kvm_memory_slot *old,
2539 enum kvm_mr_change change)
2543 /* If the basics of the memslot do not change, we do not want
2544 * to update the gmap. Every update causes several unnecessary
2545 * segment translation exceptions. This is usually handled just
2546 * fine by the normal fault handler + gmap, but it will also
2547 * cause faults on the prefix page of running guest CPUs.
2549 if (old->userspace_addr == mem->userspace_addr &&
2550 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2551 old->npages * PAGE_SIZE == mem->memory_size)
2554 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2555 mem->guest_phys_addr, mem->memory_size);
2557 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2561 static int __init kvm_s390_init(void)
2563 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2566 static void __exit kvm_s390_exit(void)
2571 module_init(kvm_s390_init);
2572 module_exit(kvm_s390_exit);
2575 * Enable autoloading of the kvm module.
2576 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2577 * since x86 takes a different approach.
2579 #include <linux/miscdevice.h>
2580 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2581 MODULE_ALIAS("devname:kvm");