2 * Copyright (C) 2012 ARM Ltd.
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 #include <linux/cpu.h>
20 #include <linux/kvm.h>
21 #include <linux/kvm_host.h>
22 #include <linux/interrupt.h>
25 #include <linux/of_address.h>
26 #include <linux/of_irq.h>
27 #include <linux/rculist.h>
28 #include <linux/uaccess.h>
30 #include <asm/kvm_emulate.h>
31 #include <asm/kvm_arm.h>
32 #include <asm/kvm_mmu.h>
33 #include <trace/events/kvm.h>
35 #include <kvm/iodev.h>
37 #define CREATE_TRACE_POINTS
41 * How the whole thing works (courtesy of Christoffer Dall):
43 * - At any time, the dist->irq_pending_on_cpu is the oracle that knows if
44 * something is pending on the CPU interface.
45 * - Interrupts that are pending on the distributor are stored on the
46 * vgic.irq_pending vgic bitmap (this bitmap is updated by both user land
47 * ioctls and guest mmio ops, and other in-kernel peripherals such as the
49 * - Every time the bitmap changes, the irq_pending_on_cpu oracle is
51 * - To calculate the oracle, we need info for each cpu from
52 * compute_pending_for_cpu, which considers:
53 * - PPI: dist->irq_pending & dist->irq_enable
54 * - SPI: dist->irq_pending & dist->irq_enable & dist->irq_spi_target
55 * - irq_spi_target is a 'formatted' version of the GICD_ITARGETSRn
56 * registers, stored on each vcpu. We only keep one bit of
57 * information per interrupt, making sure that only one vcpu can
58 * accept the interrupt.
59 * - If any of the above state changes, we must recalculate the oracle.
60 * - The same is true when injecting an interrupt, except that we only
61 * consider a single interrupt at a time. The irq_spi_cpu array
62 * contains the target CPU for each SPI.
64 * The handling of level interrupts adds some extra complexity. We
65 * need to track when the interrupt has been EOIed, so we can sample
66 * the 'line' again. This is achieved as such:
68 * - When a level interrupt is moved onto a vcpu, the corresponding
69 * bit in irq_queued is set. As long as this bit is set, the line
70 * will be ignored for further interrupts. The interrupt is injected
71 * into the vcpu with the GICH_LR_EOI bit set (generate a
72 * maintenance interrupt on EOI).
73 * - When the interrupt is EOIed, the maintenance interrupt fires,
74 * and clears the corresponding bit in irq_queued. This allows the
75 * interrupt line to be sampled again.
76 * - Note that level-triggered interrupts can also be set to pending from
77 * writes to GICD_ISPENDRn and lowering the external input line does not
78 * cause the interrupt to become inactive in such a situation.
79 * Conversely, writes to GICD_ICPENDRn do not cause the interrupt to become
80 * inactive as long as the external input line is held high.
83 * Initialization rules: there are multiple stages to the vgic
84 * initialization, both for the distributor and the CPU interfaces.
88 * - kvm_vgic_early_init(): initialization of static data that doesn't
89 * depend on any sizing information or emulation type. No allocation
92 * - vgic_init(): allocation and initialization of the generic data
93 * structures that depend on sizing information (number of CPUs,
94 * number of interrupts). Also initializes the vcpu specific data
95 * structures. Can be executed lazily for GICv2.
96 * [to be renamed to kvm_vgic_init??]
100 * - kvm_vgic_cpu_early_init(): initialization of static data that
101 * doesn't depend on any sizing information or emulation type. No
102 * allocation is allowed there.
107 static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu);
108 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
109 static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr);
110 static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr, struct vgic_lr lr_desc);
111 static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
113 static int compute_pending_for_cpu(struct kvm_vcpu *vcpu);
115 static const struct vgic_ops *vgic_ops;
116 static const struct vgic_params *vgic;
118 static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)
120 vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);
123 static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)
125 return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);
128 int kvm_vgic_map_resources(struct kvm *kvm)
130 return kvm->arch.vgic.vm_ops.map_resources(kvm, vgic);
134 * struct vgic_bitmap contains a bitmap made of unsigned longs, but
135 * extracts u32s out of them.
137 * This does not work on 64-bit BE systems, because the bitmap access
138 * will store two consecutive 32-bit words with the higher-addressed
139 * register's bits at the lower index and the lower-addressed register's
140 * bits at the higher index.
142 * Therefore, swizzle the register index when accessing the 32-bit word
143 * registers to access the right register's value.
145 #if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 64
146 #define REG_OFFSET_SWIZZLE 1
148 #define REG_OFFSET_SWIZZLE 0
151 static int vgic_init_bitmap(struct vgic_bitmap *b, int nr_cpus, int nr_irqs)
155 nr_longs = nr_cpus + BITS_TO_LONGS(nr_irqs - VGIC_NR_PRIVATE_IRQS);
157 b->private = kzalloc(sizeof(unsigned long) * nr_longs, GFP_KERNEL);
161 b->shared = b->private + nr_cpus;
166 static void vgic_free_bitmap(struct vgic_bitmap *b)
174 * Call this function to convert a u64 value to an unsigned long * bitmask
175 * in a way that works on both 32-bit and 64-bit LE and BE platforms.
177 * Warning: Calling this function may modify *val.
179 static unsigned long *u64_to_bitmask(u64 *val)
181 #if defined(CONFIG_CPU_BIG_ENDIAN) && BITS_PER_LONG == 32
182 *val = (*val >> 32) | (*val << 32);
184 return (unsigned long *)val;
187 u32 *vgic_bitmap_get_reg(struct vgic_bitmap *x, int cpuid, u32 offset)
191 return (u32 *)(x->private + cpuid) + REG_OFFSET_SWIZZLE;
193 return (u32 *)(x->shared) + ((offset - 1) ^ REG_OFFSET_SWIZZLE);
196 static int vgic_bitmap_get_irq_val(struct vgic_bitmap *x,
199 if (irq < VGIC_NR_PRIVATE_IRQS)
200 return test_bit(irq, x->private + cpuid);
202 return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);
205 void vgic_bitmap_set_irq_val(struct vgic_bitmap *x, int cpuid,
210 if (irq < VGIC_NR_PRIVATE_IRQS) {
211 reg = x->private + cpuid;
214 irq -= VGIC_NR_PRIVATE_IRQS;
223 static unsigned long *vgic_bitmap_get_cpu_map(struct vgic_bitmap *x, int cpuid)
225 return x->private + cpuid;
228 unsigned long *vgic_bitmap_get_shared_map(struct vgic_bitmap *x)
233 static int vgic_init_bytemap(struct vgic_bytemap *x, int nr_cpus, int nr_irqs)
237 size = nr_cpus * VGIC_NR_PRIVATE_IRQS;
238 size += nr_irqs - VGIC_NR_PRIVATE_IRQS;
240 x->private = kzalloc(size, GFP_KERNEL);
244 x->shared = x->private + nr_cpus * VGIC_NR_PRIVATE_IRQS / sizeof(u32);
248 static void vgic_free_bytemap(struct vgic_bytemap *b)
255 u32 *vgic_bytemap_get_reg(struct vgic_bytemap *x, int cpuid, u32 offset)
259 if (offset < VGIC_NR_PRIVATE_IRQS) {
261 offset += cpuid * VGIC_NR_PRIVATE_IRQS;
264 offset -= VGIC_NR_PRIVATE_IRQS;
267 return reg + (offset / sizeof(u32));
270 #define VGIC_CFG_LEVEL 0
271 #define VGIC_CFG_EDGE 1
273 static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)
275 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
278 irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);
279 return irq_val == VGIC_CFG_EDGE;
282 static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)
284 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
286 return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);
289 static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)
291 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
293 return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);
296 static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)
298 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
300 return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);
303 static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)
305 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
307 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);
310 static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)
312 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
314 vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);
317 static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)
319 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
321 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);
324 static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)
326 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
328 vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);
331 static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)
333 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
335 return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);
338 static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)
340 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
342 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);
345 static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)
347 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
349 vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);
352 static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)
354 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
356 return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);
359 static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)
361 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
363 vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);
364 if (!vgic_dist_irq_get_level(vcpu, irq)) {
365 vgic_dist_irq_clear_pending(vcpu, irq);
366 if (!compute_pending_for_cpu(vcpu))
367 clear_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
371 static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)
373 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
375 return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);
378 void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)
380 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
382 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);
385 void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)
387 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
389 vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);
392 static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)
394 if (irq < VGIC_NR_PRIVATE_IRQS)
395 set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
397 set_bit(irq - VGIC_NR_PRIVATE_IRQS,
398 vcpu->arch.vgic_cpu.pending_shared);
401 void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)
403 if (irq < VGIC_NR_PRIVATE_IRQS)
404 clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);
406 clear_bit(irq - VGIC_NR_PRIVATE_IRQS,
407 vcpu->arch.vgic_cpu.pending_shared);
410 static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)
412 return !vgic_irq_is_queued(vcpu, irq);
416 * vgic_reg_access - access vgic register
417 * @mmio: pointer to the data describing the mmio access
418 * @reg: pointer to the virtual backing of vgic distributor data
419 * @offset: least significant 2 bits used for word offset
420 * @mode: ACCESS_ mode (see defines above)
422 * Helper to make vgic register access easier using one of the access
423 * modes defined for vgic register access
424 * (read,raz,write-ignored,setbit,clearbit,write)
426 void vgic_reg_access(struct kvm_exit_mmio *mmio, u32 *reg,
427 phys_addr_t offset, int mode)
429 int word_offset = (offset & 3) * 8;
430 u32 mask = (1UL << (mmio->len * 8)) - 1;
434 * Any alignment fault should have been delivered to the guest
435 * directly (ARM ARM B3.12.7 "Prioritization of aborts").
441 BUG_ON(mode != (ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED));
445 if (mmio->is_write) {
446 u32 data = mmio_data_read(mmio, mask) << word_offset;
447 switch (ACCESS_WRITE_MASK(mode)) {
448 case ACCESS_WRITE_IGNORED:
451 case ACCESS_WRITE_SETBIT:
455 case ACCESS_WRITE_CLEARBIT:
459 case ACCESS_WRITE_VALUE:
460 regval = (regval & ~(mask << word_offset)) | data;
465 switch (ACCESS_READ_MASK(mode)) {
466 case ACCESS_READ_RAZ:
470 case ACCESS_READ_VALUE:
471 mmio_data_write(mmio, mask, regval >> word_offset);
476 bool handle_mmio_raz_wi(struct kvm_vcpu *vcpu, struct kvm_exit_mmio *mmio,
479 vgic_reg_access(mmio, NULL, offset,
480 ACCESS_READ_RAZ | ACCESS_WRITE_IGNORED);
484 bool vgic_handle_enable_reg(struct kvm *kvm, struct kvm_exit_mmio *mmio,
485 phys_addr_t offset, int vcpu_id, int access)
488 int mode = ACCESS_READ_VALUE | access;
489 struct kvm_vcpu *target_vcpu = kvm_get_vcpu(kvm, vcpu_id);
491 reg = vgic_bitmap_get_reg(&kvm->arch.vgic.irq_enabled, vcpu_id, offset);
492 vgic_reg_access(mmio, reg, offset, mode);
493 if (mmio->is_write) {
494 if (access & ACCESS_WRITE_CLEARBIT) {
495 if (offset < 4) /* Force SGI enabled */
497 vgic_retire_disabled_irqs(target_vcpu);
499 vgic_update_state(kvm);
506 bool vgic_handle_set_pending_reg(struct kvm *kvm,
507 struct kvm_exit_mmio *mmio,
508 phys_addr_t offset, int vcpu_id)
512 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT;
513 struct vgic_dist *dist = &kvm->arch.vgic;
515 reg = vgic_bitmap_get_reg(&dist->irq_cfg, vcpu_id, offset);
516 level_mask = (~(*reg));
518 /* Mark both level and edge triggered irqs as pending */
519 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
521 vgic_reg_access(mmio, reg, offset, mode);
523 if (mmio->is_write) {
524 /* Set the soft-pending flag only for level-triggered irqs */
525 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
527 vgic_reg_access(mmio, reg, offset, mode);
530 /* Ignore writes to SGIs */
533 *reg |= orig & 0xffff;
536 vgic_update_state(kvm);
543 bool vgic_handle_clear_pending_reg(struct kvm *kvm,
544 struct kvm_exit_mmio *mmio,
545 phys_addr_t offset, int vcpu_id)
549 int mode = ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT;
550 struct vgic_dist *dist = &kvm->arch.vgic;
552 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
554 vgic_reg_access(mmio, reg, offset, mode);
555 if (mmio->is_write) {
556 /* Re-set level triggered level-active interrupts */
557 level_active = vgic_bitmap_get_reg(&dist->irq_level,
559 reg = vgic_bitmap_get_reg(&dist->irq_pending, vcpu_id, offset);
560 *reg |= *level_active;
562 /* Ignore writes to SGIs */
565 *reg |= orig & 0xffff;
568 /* Clear soft-pending flags */
569 reg = vgic_bitmap_get_reg(&dist->irq_soft_pend,
571 vgic_reg_access(mmio, reg, offset, mode);
573 vgic_update_state(kvm);
579 bool vgic_handle_set_active_reg(struct kvm *kvm,
580 struct kvm_exit_mmio *mmio,
581 phys_addr_t offset, int vcpu_id)
584 struct vgic_dist *dist = &kvm->arch.vgic;
586 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
587 vgic_reg_access(mmio, reg, offset,
588 ACCESS_READ_VALUE | ACCESS_WRITE_SETBIT);
590 if (mmio->is_write) {
591 vgic_update_state(kvm);
598 bool vgic_handle_clear_active_reg(struct kvm *kvm,
599 struct kvm_exit_mmio *mmio,
600 phys_addr_t offset, int vcpu_id)
603 struct vgic_dist *dist = &kvm->arch.vgic;
605 reg = vgic_bitmap_get_reg(&dist->irq_active, vcpu_id, offset);
606 vgic_reg_access(mmio, reg, offset,
607 ACCESS_READ_VALUE | ACCESS_WRITE_CLEARBIT);
609 if (mmio->is_write) {
610 vgic_update_state(kvm);
617 static u32 vgic_cfg_expand(u16 val)
623 * Turn a 16bit value like abcd...mnop into a 32bit word
624 * a0b0c0d0...m0n0o0p0, which is what the HW cfg register is.
626 for (i = 0; i < 16; i++)
627 res |= ((val >> i) & VGIC_CFG_EDGE) << (2 * i + 1);
632 static u16 vgic_cfg_compress(u32 val)
638 * Turn a 32bit word a0b0c0d0...m0n0o0p0 into 16bit value like
639 * abcd...mnop which is what we really care about.
641 for (i = 0; i < 16; i++)
642 res |= ((val >> (i * 2 + 1)) & VGIC_CFG_EDGE) << i;
648 * The distributor uses 2 bits per IRQ for the CFG register, but the
649 * LSB is always 0. As such, we only keep the upper bit, and use the
650 * two above functions to compress/expand the bits
652 bool vgic_handle_cfg_reg(u32 *reg, struct kvm_exit_mmio *mmio,
662 val = vgic_cfg_expand(val);
663 vgic_reg_access(mmio, &val, offset,
664 ACCESS_READ_VALUE | ACCESS_WRITE_VALUE);
665 if (mmio->is_write) {
666 /* Ignore writes to read-only SGI and PPI bits */
670 val = vgic_cfg_compress(val);
675 *reg &= 0xffff << 16;
684 * vgic_unqueue_irqs - move pending/active IRQs from LRs to the distributor
685 * @vgic_cpu: Pointer to the vgic_cpu struct holding the LRs
687 * Move any IRQs that have already been assigned to LRs back to the
688 * emulated distributor state so that the complete emulated state can be read
689 * from the main emulation structures without investigating the LRs.
691 void vgic_unqueue_irqs(struct kvm_vcpu *vcpu)
693 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
696 for_each_set_bit(i, vgic_cpu->lr_used, vgic_cpu->nr_lr) {
697 struct vgic_lr lr = vgic_get_lr(vcpu, i);
700 * There are three options for the state bits:
704 * 11: pending and active
706 BUG_ON(!(lr.state & LR_STATE_MASK));
708 /* Reestablish SGI source for pending and active IRQs */
709 if (lr.irq < VGIC_NR_SGIS)
710 add_sgi_source(vcpu, lr.irq, lr.source);
713 * If the LR holds an active (10) or a pending and active (11)
714 * interrupt then move the active state to the
715 * distributor tracking bit.
717 if (lr.state & LR_STATE_ACTIVE) {
718 vgic_irq_set_active(vcpu, lr.irq);
719 lr.state &= ~LR_STATE_ACTIVE;
723 * Reestablish the pending state on the distributor and the
724 * CPU interface. It may have already been pending, but that
725 * is fine, then we are only setting a few bits that were
728 if (lr.state & LR_STATE_PENDING) {
729 vgic_dist_irq_set_pending(vcpu, lr.irq);
730 lr.state &= ~LR_STATE_PENDING;
733 vgic_set_lr(vcpu, i, lr);
736 * Mark the LR as free for other use.
738 BUG_ON(lr.state & LR_STATE_MASK);
739 vgic_retire_lr(i, lr.irq, vcpu);
740 vgic_irq_clear_queued(vcpu, lr.irq);
742 /* Finally update the VGIC state. */
743 vgic_update_state(vcpu->kvm);
748 struct vgic_io_range *vgic_find_range(const struct vgic_io_range *ranges,
749 int len, gpa_t offset)
751 while (ranges->len) {
752 if (offset >= ranges->base &&
753 (offset + len) <= (ranges->base + ranges->len))
761 static bool vgic_validate_access(const struct vgic_dist *dist,
762 const struct vgic_io_range *range,
763 unsigned long offset)
767 if (!range->bits_per_irq)
768 return true; /* Not an irq-based access */
770 irq = offset * 8 / range->bits_per_irq;
771 if (irq >= dist->nr_irqs)
778 * Call the respective handler function for the given range.
779 * We split up any 64 bit accesses into two consecutive 32 bit
780 * handler calls and merge the result afterwards.
781 * We do this in a little endian fashion regardless of the host's
782 * or guest's endianness, because the GIC is always LE and the rest of
783 * the code (vgic_reg_access) also puts it in a LE fashion already.
784 * At this point we have already identified the handle function, so
785 * range points to that one entry and offset is relative to this.
787 static bool call_range_handler(struct kvm_vcpu *vcpu,
788 struct kvm_exit_mmio *mmio,
789 unsigned long offset,
790 const struct vgic_io_range *range)
792 struct kvm_exit_mmio mmio32;
795 if (likely(mmio->len <= 4))
796 return range->handle_mmio(vcpu, mmio, offset);
799 * Any access bigger than 4 bytes (that we currently handle in KVM)
800 * is actually 8 bytes long, caused by a 64-bit access
804 mmio32.is_write = mmio->is_write;
805 mmio32.private = mmio->private;
807 mmio32.phys_addr = mmio->phys_addr + 4;
808 mmio32.data = &((u32 *)mmio->data)[1];
809 ret = range->handle_mmio(vcpu, &mmio32, offset + 4);
811 mmio32.phys_addr = mmio->phys_addr;
812 mmio32.data = &((u32 *)mmio->data)[0];
813 ret |= range->handle_mmio(vcpu, &mmio32, offset);
819 * vgic_handle_mmio_access - handle an in-kernel MMIO access
820 * This is called by the read/write KVM IO device wrappers below.
821 * @vcpu: pointer to the vcpu performing the access
822 * @this: pointer to the KVM IO device in charge
823 * @addr: guest physical address of the access
824 * @len: size of the access
825 * @val: pointer to the data region
826 * @is_write: read or write access
828 * returns true if the MMIO access could be performed
830 static int vgic_handle_mmio_access(struct kvm_vcpu *vcpu,
831 struct kvm_io_device *this, gpa_t addr,
832 int len, void *val, bool is_write)
834 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
835 struct vgic_io_device *iodev = container_of(this,
836 struct vgic_io_device, dev);
837 struct kvm_run *run = vcpu->run;
838 const struct vgic_io_range *range;
839 struct kvm_exit_mmio mmio;
843 offset = addr - iodev->addr;
844 range = vgic_find_range(iodev->reg_ranges, len, offset);
845 if (unlikely(!range || !range->handle_mmio)) {
846 pr_warn("Unhandled access %d %08llx %d\n", is_write, addr, len);
850 mmio.phys_addr = addr;
852 mmio.is_write = is_write;
854 mmio.private = iodev->redist_vcpu;
856 spin_lock(&dist->lock);
857 offset -= range->base;
858 if (vgic_validate_access(dist, range, offset)) {
859 updated_state = call_range_handler(vcpu, &mmio, offset, range);
863 updated_state = false;
865 spin_unlock(&dist->lock);
866 run->mmio.is_write = is_write;
868 run->mmio.phys_addr = addr;
869 memcpy(run->mmio.data, val, len);
871 kvm_handle_mmio_return(vcpu, run);
874 vgic_kick_vcpus(vcpu->kvm);
879 static int vgic_handle_mmio_read(struct kvm_vcpu *vcpu,
880 struct kvm_io_device *this,
881 gpa_t addr, int len, void *val)
883 return vgic_handle_mmio_access(vcpu, this, addr, len, val, false);
886 static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu,
887 struct kvm_io_device *this,
888 gpa_t addr, int len, const void *val)
890 return vgic_handle_mmio_access(vcpu, this, addr, len, (void *)val,
894 struct kvm_io_device_ops vgic_io_ops = {
895 .read = vgic_handle_mmio_read,
896 .write = vgic_handle_mmio_write,
900 * vgic_register_kvm_io_dev - register VGIC register frame on the KVM I/O bus
901 * @kvm: The VM structure pointer
902 * @base: The (guest) base address for the register frame
903 * @len: Length of the register frame window
904 * @ranges: Describing the handler functions for each register
905 * @redist_vcpu_id: The VCPU ID to pass on to the handlers on call
906 * @iodev: Points to memory to be passed on to the handler
908 * @iodev stores the parameters of this function to be usable by the handler
909 * respectively the dispatcher function (since the KVM I/O bus framework lacks
910 * an opaque parameter). Initialization is done in this function, but the
911 * reference should be valid and unique for the whole VGIC lifetime.
912 * If the register frame is not mapped for a specific VCPU, pass -1 to
915 int vgic_register_kvm_io_dev(struct kvm *kvm, gpa_t base, int len,
916 const struct vgic_io_range *ranges,
918 struct vgic_io_device *iodev)
920 struct kvm_vcpu *vcpu = NULL;
923 if (redist_vcpu_id >= 0)
924 vcpu = kvm_get_vcpu(kvm, redist_vcpu_id);
928 iodev->reg_ranges = ranges;
929 iodev->redist_vcpu = vcpu;
931 kvm_iodevice_init(&iodev->dev, &vgic_io_ops);
933 mutex_lock(&kvm->slots_lock);
935 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, base, len,
937 mutex_unlock(&kvm->slots_lock);
939 /* Mark the iodev as invalid if registration fails. */
941 iodev->dev.ops = NULL;
946 static int vgic_nr_shared_irqs(struct vgic_dist *dist)
948 return dist->nr_irqs - VGIC_NR_PRIVATE_IRQS;
951 static int compute_active_for_cpu(struct kvm_vcpu *vcpu)
953 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
954 unsigned long *active, *enabled, *act_percpu, *act_shared;
955 unsigned long active_private, active_shared;
956 int nr_shared = vgic_nr_shared_irqs(dist);
959 vcpu_id = vcpu->vcpu_id;
960 act_percpu = vcpu->arch.vgic_cpu.active_percpu;
961 act_shared = vcpu->arch.vgic_cpu.active_shared;
963 active = vgic_bitmap_get_cpu_map(&dist->irq_active, vcpu_id);
964 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
965 bitmap_and(act_percpu, active, enabled, VGIC_NR_PRIVATE_IRQS);
967 active = vgic_bitmap_get_shared_map(&dist->irq_active);
968 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
969 bitmap_and(act_shared, active, enabled, nr_shared);
970 bitmap_and(act_shared, act_shared,
971 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
974 active_private = find_first_bit(act_percpu, VGIC_NR_PRIVATE_IRQS);
975 active_shared = find_first_bit(act_shared, nr_shared);
977 return (active_private < VGIC_NR_PRIVATE_IRQS ||
978 active_shared < nr_shared);
981 static int compute_pending_for_cpu(struct kvm_vcpu *vcpu)
983 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
984 unsigned long *pending, *enabled, *pend_percpu, *pend_shared;
985 unsigned long pending_private, pending_shared;
986 int nr_shared = vgic_nr_shared_irqs(dist);
989 vcpu_id = vcpu->vcpu_id;
990 pend_percpu = vcpu->arch.vgic_cpu.pending_percpu;
991 pend_shared = vcpu->arch.vgic_cpu.pending_shared;
993 if (!dist->enabled) {
994 bitmap_zero(pend_percpu, VGIC_NR_PRIVATE_IRQS);
995 bitmap_zero(pend_shared, nr_shared);
999 pending = vgic_bitmap_get_cpu_map(&dist->irq_pending, vcpu_id);
1000 enabled = vgic_bitmap_get_cpu_map(&dist->irq_enabled, vcpu_id);
1001 bitmap_and(pend_percpu, pending, enabled, VGIC_NR_PRIVATE_IRQS);
1003 pending = vgic_bitmap_get_shared_map(&dist->irq_pending);
1004 enabled = vgic_bitmap_get_shared_map(&dist->irq_enabled);
1005 bitmap_and(pend_shared, pending, enabled, nr_shared);
1006 bitmap_and(pend_shared, pend_shared,
1007 vgic_bitmap_get_shared_map(&dist->irq_spi_target[vcpu_id]),
1010 pending_private = find_first_bit(pend_percpu, VGIC_NR_PRIVATE_IRQS);
1011 pending_shared = find_first_bit(pend_shared, nr_shared);
1012 return (pending_private < VGIC_NR_PRIVATE_IRQS ||
1013 pending_shared < vgic_nr_shared_irqs(dist));
1017 * Update the interrupt state and determine which CPUs have pending
1018 * or active interrupts. Must be called with distributor lock held.
1020 void vgic_update_state(struct kvm *kvm)
1022 struct vgic_dist *dist = &kvm->arch.vgic;
1023 struct kvm_vcpu *vcpu;
1026 kvm_for_each_vcpu(c, vcpu, kvm) {
1027 if (compute_pending_for_cpu(vcpu))
1028 set_bit(c, dist->irq_pending_on_cpu);
1030 if (compute_active_for_cpu(vcpu))
1031 set_bit(c, dist->irq_active_on_cpu);
1033 clear_bit(c, dist->irq_active_on_cpu);
1037 static struct vgic_lr vgic_get_lr(const struct kvm_vcpu *vcpu, int lr)
1039 return vgic_ops->get_lr(vcpu, lr);
1042 static void vgic_set_lr(struct kvm_vcpu *vcpu, int lr,
1045 vgic_ops->set_lr(vcpu, lr, vlr);
1048 static void vgic_sync_lr_elrsr(struct kvm_vcpu *vcpu, int lr,
1051 vgic_ops->sync_lr_elrsr(vcpu, lr, vlr);
1054 static inline u64 vgic_get_elrsr(struct kvm_vcpu *vcpu)
1056 return vgic_ops->get_elrsr(vcpu);
1059 static inline u64 vgic_get_eisr(struct kvm_vcpu *vcpu)
1061 return vgic_ops->get_eisr(vcpu);
1064 static inline void vgic_clear_eisr(struct kvm_vcpu *vcpu)
1066 vgic_ops->clear_eisr(vcpu);
1069 static inline u32 vgic_get_interrupt_status(struct kvm_vcpu *vcpu)
1071 return vgic_ops->get_interrupt_status(vcpu);
1074 static inline void vgic_enable_underflow(struct kvm_vcpu *vcpu)
1076 vgic_ops->enable_underflow(vcpu);
1079 static inline void vgic_disable_underflow(struct kvm_vcpu *vcpu)
1081 vgic_ops->disable_underflow(vcpu);
1084 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1086 vgic_ops->get_vmcr(vcpu, vmcr);
1089 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
1091 vgic_ops->set_vmcr(vcpu, vmcr);
1094 static inline void vgic_enable(struct kvm_vcpu *vcpu)
1096 vgic_ops->enable(vcpu);
1099 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu)
1101 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1102 struct vgic_lr vlr = vgic_get_lr(vcpu, lr_nr);
1105 * We must transfer the pending state back to the distributor before
1106 * retiring the LR, otherwise we may loose edge-triggered interrupts.
1108 if (vlr.state & LR_STATE_PENDING) {
1109 vgic_dist_irq_set_pending(vcpu, irq);
1114 vgic_set_lr(vcpu, lr_nr, vlr);
1115 clear_bit(lr_nr, vgic_cpu->lr_used);
1116 vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;
1117 vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1121 * An interrupt may have been disabled after being made pending on the
1122 * CPU interface (the classic case is a timer running while we're
1123 * rebooting the guest - the interrupt would kick as soon as the CPU
1124 * interface gets enabled, with deadly consequences).
1126 * The solution is to examine already active LRs, and check the
1127 * interrupt is still enabled. If not, just retire it.
1129 static void vgic_retire_disabled_irqs(struct kvm_vcpu *vcpu)
1131 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1134 for_each_set_bit(lr, vgic_cpu->lr_used, vgic->nr_lr) {
1135 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1137 if (!vgic_irq_is_enabled(vcpu, vlr.irq)) {
1138 vgic_retire_lr(lr, vlr.irq, vcpu);
1139 if (vgic_irq_is_queued(vcpu, vlr.irq))
1140 vgic_irq_clear_queued(vcpu, vlr.irq);
1145 static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,
1146 int lr_nr, struct vgic_lr vlr)
1148 if (vgic_irq_is_active(vcpu, irq)) {
1149 vlr.state |= LR_STATE_ACTIVE;
1150 kvm_debug("Set active, clear distributor: 0x%x\n", vlr.state);
1151 vgic_irq_clear_active(vcpu, irq);
1152 vgic_update_state(vcpu->kvm);
1154 WARN_ON(!vgic_dist_irq_is_pending(vcpu, irq));
1155 vlr.state |= LR_STATE_PENDING;
1156 kvm_debug("Set pending: 0x%x\n", vlr.state);
1159 if (!vgic_irq_is_edge(vcpu, irq))
1160 vlr.state |= LR_EOI_INT;
1162 if (vlr.irq >= VGIC_NR_SGIS) {
1163 struct irq_phys_map *map;
1164 map = vgic_irq_map_search(vcpu, irq);
1167 vlr.hwirq = map->phys_irq;
1169 vlr.state &= ~LR_EOI_INT;
1172 * Make sure we're not going to sample this
1173 * again, as a HW-backed interrupt cannot be
1174 * in the PENDING_ACTIVE stage.
1176 vgic_irq_set_queued(vcpu, irq);
1180 vgic_set_lr(vcpu, lr_nr, vlr);
1181 vgic_sync_lr_elrsr(vcpu, lr_nr, vlr);
1185 * Queue an interrupt to a CPU virtual interface. Return true on success,
1186 * or false if it wasn't possible to queue it.
1187 * sgi_source must be zero for any non-SGI interrupts.
1189 bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)
1191 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1192 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1196 /* Sanitize the input... */
1197 BUG_ON(sgi_source_id & ~7);
1198 BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);
1199 BUG_ON(irq >= dist->nr_irqs);
1201 kvm_debug("Queue IRQ%d\n", irq);
1203 lr = vgic_cpu->vgic_irq_lr_map[irq];
1205 /* Do we have an active interrupt for the same CPUID? */
1206 if (lr != LR_EMPTY) {
1207 vlr = vgic_get_lr(vcpu, lr);
1208 if (vlr.source == sgi_source_id) {
1209 kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);
1210 BUG_ON(!test_bit(lr, vgic_cpu->lr_used));
1211 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1216 /* Try to use another LR for this interrupt */
1217 lr = find_first_zero_bit((unsigned long *)vgic_cpu->lr_used,
1219 if (lr >= vgic->nr_lr)
1222 kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);
1223 vgic_cpu->vgic_irq_lr_map[irq] = lr;
1224 set_bit(lr, vgic_cpu->lr_used);
1227 vlr.source = sgi_source_id;
1229 vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);
1234 static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)
1236 if (!vgic_can_sample_irq(vcpu, irq))
1237 return true; /* level interrupt, already queued */
1239 if (vgic_queue_irq(vcpu, 0, irq)) {
1240 if (vgic_irq_is_edge(vcpu, irq)) {
1241 vgic_dist_irq_clear_pending(vcpu, irq);
1242 vgic_cpu_irq_clear(vcpu, irq);
1244 vgic_irq_set_queued(vcpu, irq);
1254 * Fill the list registers with pending interrupts before running the
1257 static void __kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1259 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1260 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1261 unsigned long *pa_percpu, *pa_shared;
1264 int nr_shared = vgic_nr_shared_irqs(dist);
1266 vcpu_id = vcpu->vcpu_id;
1268 pa_percpu = vcpu->arch.vgic_cpu.pend_act_percpu;
1269 pa_shared = vcpu->arch.vgic_cpu.pend_act_shared;
1271 bitmap_or(pa_percpu, vgic_cpu->pending_percpu, vgic_cpu->active_percpu,
1272 VGIC_NR_PRIVATE_IRQS);
1273 bitmap_or(pa_shared, vgic_cpu->pending_shared, vgic_cpu->active_shared,
1276 * We may not have any pending interrupt, or the interrupts
1277 * may have been serviced from another vcpu. In all cases,
1280 if (!kvm_vgic_vcpu_pending_irq(vcpu) && !kvm_vgic_vcpu_active_irq(vcpu))
1284 for_each_set_bit(i, pa_percpu, VGIC_NR_SGIS) {
1285 if (!queue_sgi(vcpu, i))
1290 for_each_set_bit_from(i, pa_percpu, VGIC_NR_PRIVATE_IRQS) {
1291 if (!vgic_queue_hwirq(vcpu, i))
1296 for_each_set_bit(i, pa_shared, nr_shared) {
1297 if (!vgic_queue_hwirq(vcpu, i + VGIC_NR_PRIVATE_IRQS))
1306 vgic_enable_underflow(vcpu);
1308 vgic_disable_underflow(vcpu);
1310 * We're about to run this VCPU, and we've consumed
1311 * everything the distributor had in store for
1312 * us. Claim we don't have anything pending. We'll
1313 * adjust that if needed while exiting.
1315 clear_bit(vcpu_id, dist->irq_pending_on_cpu);
1319 static int process_queued_irq(struct kvm_vcpu *vcpu,
1320 int lr, struct vgic_lr vlr)
1325 * If the IRQ was EOIed (called from vgic_process_maintenance) or it
1326 * went from active to non-active (called from vgic_sync_hwirq) it was
1327 * also ACKed and we we therefore assume we can clear the soft pending
1328 * state (should it had been set) for this interrupt.
1330 * Note: if the IRQ soft pending state was set after the IRQ was
1331 * acked, it actually shouldn't be cleared, but we have no way of
1332 * knowing that unless we start trapping ACKs when the soft-pending
1335 vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);
1338 * Tell the gic to start sampling this interrupt again.
1340 vgic_irq_clear_queued(vcpu, vlr.irq);
1342 /* Any additional pending interrupt? */
1343 if (vgic_irq_is_edge(vcpu, vlr.irq)) {
1344 BUG_ON(!(vlr.state & LR_HW));
1345 pending = vgic_dist_irq_is_pending(vcpu, vlr.irq);
1347 if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {
1348 vgic_cpu_irq_set(vcpu, vlr.irq);
1351 vgic_dist_irq_clear_pending(vcpu, vlr.irq);
1352 vgic_cpu_irq_clear(vcpu, vlr.irq);
1357 * Despite being EOIed, the LR may not have
1358 * been marked as empty.
1362 vgic_set_lr(vcpu, lr, vlr);
1364 vgic_sync_lr_elrsr(vcpu, lr, vlr);
1369 static bool vgic_process_maintenance(struct kvm_vcpu *vcpu)
1371 u32 status = vgic_get_interrupt_status(vcpu);
1372 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1373 struct kvm *kvm = vcpu->kvm;
1374 int level_pending = 0;
1376 kvm_debug("STATUS = %08x\n", status);
1378 if (status & INT_STATUS_EOI) {
1380 * Some level interrupts have been EOIed. Clear their
1383 u64 eisr = vgic_get_eisr(vcpu);
1384 unsigned long *eisr_ptr = u64_to_bitmask(&eisr);
1387 for_each_set_bit(lr, eisr_ptr, vgic->nr_lr) {
1388 struct vgic_lr vlr = vgic_get_lr(vcpu, lr);
1390 WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));
1391 WARN_ON(vlr.state & LR_STATE_MASK);
1395 * kvm_notify_acked_irq calls kvm_set_irq()
1396 * to reset the IRQ level, which grabs the dist->lock
1397 * so we call this before taking the dist->lock.
1399 kvm_notify_acked_irq(kvm, 0,
1400 vlr.irq - VGIC_NR_PRIVATE_IRQS);
1402 spin_lock(&dist->lock);
1403 level_pending |= process_queued_irq(vcpu, lr, vlr);
1404 spin_unlock(&dist->lock);
1408 if (status & INT_STATUS_UNDERFLOW)
1409 vgic_disable_underflow(vcpu);
1412 * In the next iterations of the vcpu loop, if we sync the vgic state
1413 * after flushing it, but before entering the guest (this happens for
1414 * pending signals and vmid rollovers), then make sure we don't pick
1415 * up any old maintenance interrupts here.
1417 vgic_clear_eisr(vcpu);
1419 return level_pending;
1423 * Save the physical active state, and reset it to inactive.
1425 * Return true if there's a pending forwarded interrupt to queue.
1427 static bool vgic_sync_hwirq(struct kvm_vcpu *vcpu, int lr, struct vgic_lr vlr)
1429 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1430 struct irq_phys_map *map;
1435 if (!(vlr.state & LR_HW))
1438 map = vgic_irq_map_search(vcpu, vlr.irq);
1441 ret = irq_get_irqchip_state(map->irq,
1442 IRQCHIP_STATE_ACTIVE,
1450 spin_lock(&dist->lock);
1451 level_pending = process_queued_irq(vcpu, lr, vlr);
1452 spin_unlock(&dist->lock);
1453 return level_pending;
1456 /* Sync back the VGIC state after a guest run */
1457 static void __kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1459 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1460 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1462 unsigned long *elrsr_ptr;
1466 level_pending = vgic_process_maintenance(vcpu);
1467 elrsr = vgic_get_elrsr(vcpu);
1468 elrsr_ptr = u64_to_bitmask(&elrsr);
1470 /* Deal with HW interrupts, and clear mappings for empty LRs */
1471 for (lr = 0; lr < vgic->nr_lr; lr++) {
1474 if (!test_bit(lr, vgic_cpu->lr_used))
1477 vlr = vgic_get_lr(vcpu, lr);
1478 if (vgic_sync_hwirq(vcpu, lr, vlr))
1479 level_pending = true;
1481 if (!test_bit(lr, elrsr_ptr))
1484 clear_bit(lr, vgic_cpu->lr_used);
1486 BUG_ON(vlr.irq >= dist->nr_irqs);
1487 vgic_cpu->vgic_irq_lr_map[vlr.irq] = LR_EMPTY;
1490 /* Check if we still have something up our sleeve... */
1491 pending = find_first_zero_bit(elrsr_ptr, vgic->nr_lr);
1492 if (level_pending || pending < vgic->nr_lr)
1493 set_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1496 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
1498 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1500 if (!irqchip_in_kernel(vcpu->kvm))
1503 spin_lock(&dist->lock);
1504 __kvm_vgic_flush_hwstate(vcpu);
1505 spin_unlock(&dist->lock);
1508 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
1510 if (!irqchip_in_kernel(vcpu->kvm))
1513 __kvm_vgic_sync_hwstate(vcpu);
1516 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
1518 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1520 if (!irqchip_in_kernel(vcpu->kvm))
1523 return test_bit(vcpu->vcpu_id, dist->irq_pending_on_cpu);
1526 int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu)
1528 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1530 if (!irqchip_in_kernel(vcpu->kvm))
1533 return test_bit(vcpu->vcpu_id, dist->irq_active_on_cpu);
1537 void vgic_kick_vcpus(struct kvm *kvm)
1539 struct kvm_vcpu *vcpu;
1543 * We've injected an interrupt, time to find out who deserves
1546 kvm_for_each_vcpu(c, vcpu, kvm) {
1547 if (kvm_vgic_vcpu_pending_irq(vcpu))
1548 kvm_vcpu_kick(vcpu);
1552 static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)
1554 int edge_triggered = vgic_irq_is_edge(vcpu, irq);
1557 * Only inject an interrupt if:
1558 * - edge triggered and we have a rising edge
1559 * - level triggered and we change level
1561 if (edge_triggered) {
1562 int state = vgic_dist_irq_is_pending(vcpu, irq);
1563 return level > state;
1565 int state = vgic_dist_irq_get_level(vcpu, irq);
1566 return level != state;
1570 static int vgic_update_irq_pending(struct kvm *kvm, int cpuid,
1571 struct irq_phys_map *map,
1572 unsigned int irq_num, bool level)
1574 struct vgic_dist *dist = &kvm->arch.vgic;
1575 struct kvm_vcpu *vcpu;
1576 int edge_triggered, level_triggered;
1578 bool ret = true, can_inject = true;
1580 trace_vgic_update_irq_pending(cpuid, irq_num, level);
1582 if (irq_num >= min(kvm->arch.vgic.nr_irqs, 1020))
1585 spin_lock(&dist->lock);
1587 vcpu = kvm_get_vcpu(kvm, cpuid);
1588 edge_triggered = vgic_irq_is_edge(vcpu, irq_num);
1589 level_triggered = !edge_triggered;
1591 if (!vgic_validate_injection(vcpu, irq_num, level)) {
1596 if (irq_num >= VGIC_NR_PRIVATE_IRQS) {
1597 cpuid = dist->irq_spi_cpu[irq_num - VGIC_NR_PRIVATE_IRQS];
1598 if (cpuid == VCPU_NOT_ALLOCATED) {
1599 /* Pretend we use CPU0, and prevent injection */
1603 vcpu = kvm_get_vcpu(kvm, cpuid);
1606 kvm_debug("Inject IRQ%d level %d CPU%d\n", irq_num, level, cpuid);
1609 if (level_triggered)
1610 vgic_dist_irq_set_level(vcpu, irq_num);
1611 vgic_dist_irq_set_pending(vcpu, irq_num);
1613 if (level_triggered) {
1614 vgic_dist_irq_clear_level(vcpu, irq_num);
1615 if (!vgic_dist_irq_soft_pend(vcpu, irq_num)) {
1616 vgic_dist_irq_clear_pending(vcpu, irq_num);
1617 vgic_cpu_irq_clear(vcpu, irq_num);
1618 if (!compute_pending_for_cpu(vcpu))
1619 clear_bit(cpuid, dist->irq_pending_on_cpu);
1627 enabled = vgic_irq_is_enabled(vcpu, irq_num);
1629 if (!enabled || !can_inject) {
1634 if (!vgic_can_sample_irq(vcpu, irq_num)) {
1636 * Level interrupt in progress, will be picked up
1644 vgic_cpu_irq_set(vcpu, irq_num);
1645 set_bit(cpuid, dist->irq_pending_on_cpu);
1649 spin_unlock(&dist->lock);
1652 /* kick the specified vcpu */
1653 kvm_vcpu_kick(kvm_get_vcpu(kvm, cpuid));
1659 static int vgic_lazy_init(struct kvm *kvm)
1663 if (unlikely(!vgic_initialized(kvm))) {
1665 * We only provide the automatic initialization of the VGIC
1666 * for the legacy case of a GICv2. Any other type must
1667 * be explicitly initialized once setup with the respective
1670 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
1673 mutex_lock(&kvm->lock);
1674 ret = vgic_init(kvm);
1675 mutex_unlock(&kvm->lock);
1682 * kvm_vgic_inject_irq - Inject an IRQ from a device to the vgic
1683 * @kvm: The VM structure pointer
1684 * @cpuid: The CPU for PPIs
1685 * @irq_num: The IRQ number that is assigned to the device. This IRQ
1686 * must not be mapped to a HW interrupt.
1687 * @level: Edge-triggered: true: to trigger the interrupt
1688 * false: to ignore the call
1689 * Level-sensitive true: raise the input signal
1690 * false: lower the input signal
1692 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1693 * level-sensitive interrupts. You can think of the level parameter as 1
1694 * being HIGH and 0 being LOW and all devices being active-HIGH.
1696 int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int irq_num,
1699 struct irq_phys_map *map;
1702 ret = vgic_lazy_init(kvm);
1706 map = vgic_irq_map_search(kvm_get_vcpu(kvm, cpuid), irq_num);
1710 return vgic_update_irq_pending(kvm, cpuid, NULL, irq_num, level);
1714 * kvm_vgic_inject_mapped_irq - Inject a physically mapped IRQ to the vgic
1715 * @kvm: The VM structure pointer
1716 * @cpuid: The CPU for PPIs
1717 * @map: Pointer to a irq_phys_map structure describing the mapping
1718 * @level: Edge-triggered: true: to trigger the interrupt
1719 * false: to ignore the call
1720 * Level-sensitive true: raise the input signal
1721 * false: lower the input signal
1723 * The GIC is not concerned with devices being active-LOW or active-HIGH for
1724 * level-sensitive interrupts. You can think of the level parameter as 1
1725 * being HIGH and 0 being LOW and all devices being active-HIGH.
1727 int kvm_vgic_inject_mapped_irq(struct kvm *kvm, int cpuid,
1728 struct irq_phys_map *map, bool level)
1732 ret = vgic_lazy_init(kvm);
1736 return vgic_update_irq_pending(kvm, cpuid, map, map->virt_irq, level);
1739 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
1742 * We cannot rely on the vgic maintenance interrupt to be
1743 * delivered synchronously. This means we can only use it to
1744 * exit the VM, and we perform the handling of EOIed
1745 * interrupts on the exit path (see vgic_process_maintenance).
1750 static struct list_head *vgic_get_irq_phys_map_list(struct kvm_vcpu *vcpu,
1753 if (virt_irq < VGIC_NR_PRIVATE_IRQS)
1754 return &vcpu->arch.vgic_cpu.irq_phys_map_list;
1756 return &vcpu->kvm->arch.vgic.irq_phys_map_list;
1760 * kvm_vgic_map_phys_irq - map a virtual IRQ to a physical IRQ
1761 * @vcpu: The VCPU pointer
1762 * @virt_irq: The virtual irq number
1763 * @irq: The Linux IRQ number
1765 * Establish a mapping between a guest visible irq (@virt_irq) and a
1766 * Linux irq (@irq). On injection, @virt_irq will be associated with
1767 * the physical interrupt represented by @irq. This mapping can be
1768 * established multiple times as long as the parameters are the same.
1770 * Returns a valid pointer on success, and an error pointer otherwise
1772 struct irq_phys_map *kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu,
1773 int virt_irq, int irq)
1775 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1776 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1777 struct irq_phys_map *map;
1778 struct irq_phys_map_entry *entry;
1779 struct irq_desc *desc;
1780 struct irq_data *data;
1783 desc = irq_to_desc(irq);
1785 kvm_err("%s: no interrupt descriptor\n", __func__);
1786 return ERR_PTR(-EINVAL);
1789 data = irq_desc_get_irq_data(desc);
1790 while (data->parent_data)
1791 data = data->parent_data;
1793 phys_irq = data->hwirq;
1795 /* Create a new mapping */
1796 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1798 return ERR_PTR(-ENOMEM);
1800 spin_lock(&dist->irq_phys_map_lock);
1802 /* Try to match an existing mapping */
1803 map = vgic_irq_map_search(vcpu, virt_irq);
1805 /* Make sure this mapping matches */
1806 if (map->phys_irq != phys_irq ||
1808 map = ERR_PTR(-EINVAL);
1810 /* Found an existing, valid mapping */
1815 map->virt_irq = virt_irq;
1816 map->phys_irq = phys_irq;
1819 list_add_tail_rcu(&entry->entry, root);
1822 spin_unlock(&dist->irq_phys_map_lock);
1823 /* If we've found a hit in the existing list, free the useless
1825 if (IS_ERR(map) || map != &entry->map)
1830 static struct irq_phys_map *vgic_irq_map_search(struct kvm_vcpu *vcpu,
1833 struct list_head *root = vgic_get_irq_phys_map_list(vcpu, virt_irq);
1834 struct irq_phys_map_entry *entry;
1835 struct irq_phys_map *map;
1839 list_for_each_entry_rcu(entry, root, entry) {
1841 if (map->virt_irq == virt_irq) {
1852 static void vgic_free_phys_irq_map_rcu(struct rcu_head *rcu)
1854 struct irq_phys_map_entry *entry;
1856 entry = container_of(rcu, struct irq_phys_map_entry, rcu);
1861 * kvm_vgic_unmap_phys_irq - Remove a virtual to physical IRQ mapping
1862 * @vcpu: The VCPU pointer
1863 * @map: The pointer to a mapping obtained through kvm_vgic_map_phys_irq
1865 * Remove an existing mapping between virtual and physical interrupts.
1867 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, struct irq_phys_map *map)
1869 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
1870 struct irq_phys_map_entry *entry;
1871 struct list_head *root;
1876 root = vgic_get_irq_phys_map_list(vcpu, map->virt_irq);
1878 spin_lock(&dist->irq_phys_map_lock);
1880 list_for_each_entry(entry, root, entry) {
1881 if (&entry->map == map) {
1882 list_del_rcu(&entry->entry);
1883 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1888 spin_unlock(&dist->irq_phys_map_lock);
1893 static void vgic_destroy_irq_phys_map(struct kvm *kvm, struct list_head *root)
1895 struct vgic_dist *dist = &kvm->arch.vgic;
1896 struct irq_phys_map_entry *entry;
1898 spin_lock(&dist->irq_phys_map_lock);
1900 list_for_each_entry(entry, root, entry) {
1901 list_del_rcu(&entry->entry);
1902 call_rcu(&entry->rcu, vgic_free_phys_irq_map_rcu);
1905 spin_unlock(&dist->irq_phys_map_lock);
1908 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
1910 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1912 kfree(vgic_cpu->pending_shared);
1913 kfree(vgic_cpu->active_shared);
1914 kfree(vgic_cpu->pend_act_shared);
1915 kfree(vgic_cpu->vgic_irq_lr_map);
1916 vgic_destroy_irq_phys_map(vcpu->kvm, &vgic_cpu->irq_phys_map_list);
1917 vgic_cpu->pending_shared = NULL;
1918 vgic_cpu->active_shared = NULL;
1919 vgic_cpu->pend_act_shared = NULL;
1920 vgic_cpu->vgic_irq_lr_map = NULL;
1923 static int vgic_vcpu_init_maps(struct kvm_vcpu *vcpu, int nr_irqs)
1925 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1927 int sz = (nr_irqs - VGIC_NR_PRIVATE_IRQS) / 8;
1928 vgic_cpu->pending_shared = kzalloc(sz, GFP_KERNEL);
1929 vgic_cpu->active_shared = kzalloc(sz, GFP_KERNEL);
1930 vgic_cpu->pend_act_shared = kzalloc(sz, GFP_KERNEL);
1931 vgic_cpu->vgic_irq_lr_map = kmalloc(nr_irqs, GFP_KERNEL);
1933 if (!vgic_cpu->pending_shared
1934 || !vgic_cpu->active_shared
1935 || !vgic_cpu->pend_act_shared
1936 || !vgic_cpu->vgic_irq_lr_map) {
1937 kvm_vgic_vcpu_destroy(vcpu);
1941 memset(vgic_cpu->vgic_irq_lr_map, LR_EMPTY, nr_irqs);
1944 * Store the number of LRs per vcpu, so we don't have to go
1945 * all the way to the distributor structure to find out. Only
1946 * assembly code should use this one.
1948 vgic_cpu->nr_lr = vgic->nr_lr;
1954 * kvm_vgic_vcpu_early_init - Earliest possible per-vcpu vgic init stage
1956 * No memory allocation should be performed here, only static init.
1958 void kvm_vgic_vcpu_early_init(struct kvm_vcpu *vcpu)
1960 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
1961 INIT_LIST_HEAD(&vgic_cpu->irq_phys_map_list);
1965 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
1967 * The host's GIC naturally limits the maximum amount of VCPUs a guest
1970 int kvm_vgic_get_max_vcpus(void)
1972 return vgic->max_gic_vcpus;
1975 void kvm_vgic_destroy(struct kvm *kvm)
1977 struct vgic_dist *dist = &kvm->arch.vgic;
1978 struct kvm_vcpu *vcpu;
1981 kvm_for_each_vcpu(i, vcpu, kvm)
1982 kvm_vgic_vcpu_destroy(vcpu);
1984 vgic_free_bitmap(&dist->irq_enabled);
1985 vgic_free_bitmap(&dist->irq_level);
1986 vgic_free_bitmap(&dist->irq_pending);
1987 vgic_free_bitmap(&dist->irq_soft_pend);
1988 vgic_free_bitmap(&dist->irq_queued);
1989 vgic_free_bitmap(&dist->irq_cfg);
1990 vgic_free_bytemap(&dist->irq_priority);
1991 if (dist->irq_spi_target) {
1992 for (i = 0; i < dist->nr_cpus; i++)
1993 vgic_free_bitmap(&dist->irq_spi_target[i]);
1995 kfree(dist->irq_sgi_sources);
1996 kfree(dist->irq_spi_cpu);
1997 kfree(dist->irq_spi_mpidr);
1998 kfree(dist->irq_spi_target);
1999 kfree(dist->irq_pending_on_cpu);
2000 kfree(dist->irq_active_on_cpu);
2001 vgic_destroy_irq_phys_map(kvm, &dist->irq_phys_map_list);
2002 dist->irq_sgi_sources = NULL;
2003 dist->irq_spi_cpu = NULL;
2004 dist->irq_spi_target = NULL;
2005 dist->irq_pending_on_cpu = NULL;
2006 dist->irq_active_on_cpu = NULL;
2011 * Allocate and initialize the various data structures. Must be called
2012 * with kvm->lock held!
2014 int vgic_init(struct kvm *kvm)
2016 struct vgic_dist *dist = &kvm->arch.vgic;
2017 struct kvm_vcpu *vcpu;
2018 int nr_cpus, nr_irqs;
2019 int ret, i, vcpu_id;
2021 if (vgic_initialized(kvm))
2024 nr_cpus = dist->nr_cpus = atomic_read(&kvm->online_vcpus);
2025 if (!nr_cpus) /* No vcpus? Can't be good... */
2029 * If nobody configured the number of interrupts, use the
2033 dist->nr_irqs = VGIC_NR_IRQS_LEGACY;
2035 nr_irqs = dist->nr_irqs;
2037 ret = vgic_init_bitmap(&dist->irq_enabled, nr_cpus, nr_irqs);
2038 ret |= vgic_init_bitmap(&dist->irq_level, nr_cpus, nr_irqs);
2039 ret |= vgic_init_bitmap(&dist->irq_pending, nr_cpus, nr_irqs);
2040 ret |= vgic_init_bitmap(&dist->irq_soft_pend, nr_cpus, nr_irqs);
2041 ret |= vgic_init_bitmap(&dist->irq_queued, nr_cpus, nr_irqs);
2042 ret |= vgic_init_bitmap(&dist->irq_active, nr_cpus, nr_irqs);
2043 ret |= vgic_init_bitmap(&dist->irq_cfg, nr_cpus, nr_irqs);
2044 ret |= vgic_init_bytemap(&dist->irq_priority, nr_cpus, nr_irqs);
2049 dist->irq_sgi_sources = kzalloc(nr_cpus * VGIC_NR_SGIS, GFP_KERNEL);
2050 dist->irq_spi_cpu = kzalloc(nr_irqs - VGIC_NR_PRIVATE_IRQS, GFP_KERNEL);
2051 dist->irq_spi_target = kzalloc(sizeof(*dist->irq_spi_target) * nr_cpus,
2053 dist->irq_pending_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
2055 dist->irq_active_on_cpu = kzalloc(BITS_TO_LONGS(nr_cpus) * sizeof(long),
2057 if (!dist->irq_sgi_sources ||
2058 !dist->irq_spi_cpu ||
2059 !dist->irq_spi_target ||
2060 !dist->irq_pending_on_cpu ||
2061 !dist->irq_active_on_cpu) {
2066 for (i = 0; i < nr_cpus; i++)
2067 ret |= vgic_init_bitmap(&dist->irq_spi_target[i],
2073 ret = kvm->arch.vgic.vm_ops.init_model(kvm);
2077 kvm_for_each_vcpu(vcpu_id, vcpu, kvm) {
2078 ret = vgic_vcpu_init_maps(vcpu, nr_irqs);
2080 kvm_err("VGIC: Failed to allocate vcpu memory\n");
2085 * Enable and configure all SGIs to be edge-triggere and
2086 * configure all PPIs as level-triggered.
2088 for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
2089 if (i < VGIC_NR_SGIS) {
2091 vgic_bitmap_set_irq_val(&dist->irq_enabled,
2092 vcpu->vcpu_id, i, 1);
2093 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2096 } else if (i < VGIC_NR_PRIVATE_IRQS) {
2098 vgic_bitmap_set_irq_val(&dist->irq_cfg,
2109 kvm_vgic_destroy(kvm);
2114 static int init_vgic_model(struct kvm *kvm, int type)
2117 case KVM_DEV_TYPE_ARM_VGIC_V2:
2118 vgic_v2_init_emulation(kvm);
2120 #ifdef CONFIG_ARM_GIC_V3
2121 case KVM_DEV_TYPE_ARM_VGIC_V3:
2122 vgic_v3_init_emulation(kvm);
2129 if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus)
2136 * kvm_vgic_early_init - Earliest possible vgic initialization stage
2138 * No memory allocation should be performed here, only static init.
2140 void kvm_vgic_early_init(struct kvm *kvm)
2142 spin_lock_init(&kvm->arch.vgic.lock);
2143 spin_lock_init(&kvm->arch.vgic.irq_phys_map_lock);
2144 INIT_LIST_HEAD(&kvm->arch.vgic.irq_phys_map_list);
2147 int kvm_vgic_create(struct kvm *kvm, u32 type)
2149 int i, vcpu_lock_idx = -1, ret;
2150 struct kvm_vcpu *vcpu;
2152 mutex_lock(&kvm->lock);
2154 if (irqchip_in_kernel(kvm)) {
2160 * This function is also called by the KVM_CREATE_IRQCHIP handler,
2161 * which had no chance yet to check the availability of the GICv2
2162 * emulation. So check this here again. KVM_CREATE_DEVICE does
2163 * the proper checks already.
2165 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 && !vgic->can_emulate_gicv2) {
2171 * Any time a vcpu is run, vcpu_load is called which tries to grab the
2172 * vcpu->mutex. By grabbing the vcpu->mutex of all VCPUs we ensure
2173 * that no other VCPUs are run while we create the vgic.
2176 kvm_for_each_vcpu(i, vcpu, kvm) {
2177 if (!mutex_trylock(&vcpu->mutex))
2182 kvm_for_each_vcpu(i, vcpu, kvm) {
2183 if (vcpu->arch.has_run_once)
2188 ret = init_vgic_model(kvm, type);
2192 kvm->arch.vgic.in_kernel = true;
2193 kvm->arch.vgic.vgic_model = type;
2194 kvm->arch.vgic.vctrl_base = vgic->vctrl_base;
2195 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
2196 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
2197 kvm->arch.vgic.vgic_redist_base = VGIC_ADDR_UNDEF;
2200 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
2201 vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
2202 mutex_unlock(&vcpu->mutex);
2206 mutex_unlock(&kvm->lock);
2210 static int vgic_ioaddr_overlap(struct kvm *kvm)
2212 phys_addr_t dist = kvm->arch.vgic.vgic_dist_base;
2213 phys_addr_t cpu = kvm->arch.vgic.vgic_cpu_base;
2215 if (IS_VGIC_ADDR_UNDEF(dist) || IS_VGIC_ADDR_UNDEF(cpu))
2217 if ((dist <= cpu && dist + KVM_VGIC_V2_DIST_SIZE > cpu) ||
2218 (cpu <= dist && cpu + KVM_VGIC_V2_CPU_SIZE > dist))
2223 static int vgic_ioaddr_assign(struct kvm *kvm, phys_addr_t *ioaddr,
2224 phys_addr_t addr, phys_addr_t size)
2228 if (addr & ~KVM_PHYS_MASK)
2231 if (addr & (SZ_4K - 1))
2234 if (!IS_VGIC_ADDR_UNDEF(*ioaddr))
2236 if (addr + size < addr)
2240 ret = vgic_ioaddr_overlap(kvm);
2242 *ioaddr = VGIC_ADDR_UNDEF;
2248 * kvm_vgic_addr - set or get vgic VM base addresses
2249 * @kvm: pointer to the vm struct
2250 * @type: the VGIC addr type, one of KVM_VGIC_V[23]_ADDR_TYPE_XXX
2251 * @addr: pointer to address value
2252 * @write: if true set the address in the VM address space, if false read the
2255 * Set or get the vgic base addresses for the distributor and the virtual CPU
2256 * interface in the VM physical address space. These addresses are properties
2257 * of the emulated core/SoC and therefore user space initially knows this
2260 int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write)
2263 struct vgic_dist *vgic = &kvm->arch.vgic;
2265 phys_addr_t *addr_ptr, block_size;
2266 phys_addr_t alignment;
2268 mutex_lock(&kvm->lock);
2270 case KVM_VGIC_V2_ADDR_TYPE_DIST:
2271 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2272 addr_ptr = &vgic->vgic_dist_base;
2273 block_size = KVM_VGIC_V2_DIST_SIZE;
2276 case KVM_VGIC_V2_ADDR_TYPE_CPU:
2277 type_needed = KVM_DEV_TYPE_ARM_VGIC_V2;
2278 addr_ptr = &vgic->vgic_cpu_base;
2279 block_size = KVM_VGIC_V2_CPU_SIZE;
2282 #ifdef CONFIG_ARM_GIC_V3
2283 case KVM_VGIC_V3_ADDR_TYPE_DIST:
2284 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2285 addr_ptr = &vgic->vgic_dist_base;
2286 block_size = KVM_VGIC_V3_DIST_SIZE;
2289 case KVM_VGIC_V3_ADDR_TYPE_REDIST:
2290 type_needed = KVM_DEV_TYPE_ARM_VGIC_V3;
2291 addr_ptr = &vgic->vgic_redist_base;
2292 block_size = KVM_VGIC_V3_REDIST_SIZE;
2301 if (vgic->vgic_model != type_needed) {
2307 if (!IS_ALIGNED(*addr, alignment))
2310 r = vgic_ioaddr_assign(kvm, addr_ptr, *addr,
2317 mutex_unlock(&kvm->lock);
2321 int vgic_set_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2325 switch (attr->group) {
2326 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2327 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2329 unsigned long type = (unsigned long)attr->attr;
2331 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2334 r = kvm_vgic_addr(dev->kvm, type, &addr, true);
2335 return (r == -ENODEV) ? -ENXIO : r;
2337 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2338 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2342 if (get_user(val, uaddr))
2347 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
2348 * - at most 1024 interrupts
2349 * - a multiple of 32 interrupts
2351 if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
2352 val > VGIC_MAX_IRQS ||
2356 mutex_lock(&dev->kvm->lock);
2358 if (vgic_ready(dev->kvm) || dev->kvm->arch.vgic.nr_irqs)
2361 dev->kvm->arch.vgic.nr_irqs = val;
2363 mutex_unlock(&dev->kvm->lock);
2367 case KVM_DEV_ARM_VGIC_GRP_CTRL: {
2368 switch (attr->attr) {
2369 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2370 r = vgic_init(dev->kvm);
2380 int vgic_get_common_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2384 switch (attr->group) {
2385 case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2386 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2388 unsigned long type = (unsigned long)attr->attr;
2390 r = kvm_vgic_addr(dev->kvm, type, &addr, false);
2392 return (r == -ENODEV) ? -ENXIO : r;
2394 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2398 case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
2399 u32 __user *uaddr = (u32 __user *)(long)attr->addr;
2401 r = put_user(dev->kvm->arch.vgic.nr_irqs, uaddr);
2410 int vgic_has_attr_regs(const struct vgic_io_range *ranges, phys_addr_t offset)
2412 if (vgic_find_range(ranges, 4, offset))
2418 static void vgic_init_maintenance_interrupt(void *info)
2420 enable_percpu_irq(vgic->maint_irq, 0);
2423 static int vgic_cpu_notify(struct notifier_block *self,
2424 unsigned long action, void *cpu)
2428 case CPU_STARTING_FROZEN:
2429 vgic_init_maintenance_interrupt(NULL);
2432 case CPU_DYING_FROZEN:
2433 disable_percpu_irq(vgic->maint_irq);
2440 static struct notifier_block vgic_cpu_nb = {
2441 .notifier_call = vgic_cpu_notify,
2444 static const struct of_device_id vgic_ids[] = {
2445 { .compatible = "arm,cortex-a15-gic", .data = vgic_v2_probe, },
2446 { .compatible = "arm,cortex-a7-gic", .data = vgic_v2_probe, },
2447 { .compatible = "arm,gic-400", .data = vgic_v2_probe, },
2448 { .compatible = "arm,gic-v3", .data = vgic_v3_probe, },
2452 int kvm_vgic_hyp_init(void)
2454 const struct of_device_id *matched_id;
2455 const int (*vgic_probe)(struct device_node *,const struct vgic_ops **,
2456 const struct vgic_params **);
2457 struct device_node *vgic_node;
2460 vgic_node = of_find_matching_node_and_match(NULL,
2461 vgic_ids, &matched_id);
2463 kvm_err("error: no compatible GIC node found\n");
2467 vgic_probe = matched_id->data;
2468 ret = vgic_probe(vgic_node, &vgic_ops, &vgic);
2472 ret = request_percpu_irq(vgic->maint_irq, vgic_maintenance_handler,
2473 "vgic", kvm_get_running_vcpus());
2475 kvm_err("Cannot register interrupt %d\n", vgic->maint_irq);
2479 ret = __register_cpu_notifier(&vgic_cpu_nb);
2481 kvm_err("Cannot register vgic CPU notifier\n");
2485 on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
2490 free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
2494 int kvm_irq_map_gsi(struct kvm *kvm,
2495 struct kvm_kernel_irq_routing_entry *entries,
2501 int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin)
2506 int kvm_set_irq(struct kvm *kvm, int irq_source_id,
2507 u32 irq, int level, bool line_status)
2509 unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;
2511 trace_kvm_set_irq(irq, level, irq_source_id);
2513 BUG_ON(!vgic_initialized(kvm));
2515 return kvm_vgic_inject_irq(kvm, 0, spi, level);
2518 /* MSI not implemented yet */
2519 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
2520 struct kvm *kvm, int irq_source_id,
2521 int level, bool line_status)