1 #include <linux/irqchip/arm-gic.h>
2 #include <asm/assembler.h>
4 #define VCPU_USR_REG(_reg_nr) (VCPU_USR_REGS + (_reg_nr * 4))
5 #define VCPU_USR_SP (VCPU_USR_REG(13))
6 #define VCPU_USR_LR (VCPU_USR_REG(14))
7 #define CP15_OFFSET(_cp15_reg_idx) (VCPU_CP15 + (_cp15_reg_idx * 4))
10 * Many of these macros need to access the VCPU structure, which is always
11 * held in r0. These macros should never clobber r1, as it is used to hold the
12 * exception code on the return path (except of course the macro that switches
13 * all the registers before the final jump to the VM).
15 vcpu .req r0 @ vcpu pointer always in r0
17 /* Clobbers {r2-r6} */
18 .macro store_vfp_state vfp_base
19 @ The VFPFMRX and VFPFMXR macros are the VMRS and VMSR instructions
21 @ Make sure VFP is enabled so we can touch the registers.
26 tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
28 @ If FPEXC_EX is 0, then FPINST/FPINST2 reads are upredictable, so
29 @ we only need to save them if FPEXC_EX is set.
32 VFPFMRX r5, FPINST2, ne @ vmrsne
33 bic r6, r2, #FPEXC_EX @ FPEXC_EX disable
36 VFPFSTMIA \vfp_base, r6 @ Save VFP registers
37 stm \vfp_base, {r2-r5} @ Save FPEXC, FPSCR, FPINST, FPINST2
40 /* Assume FPEXC_EN is on and FPEXC_EX is off, clobbers {r2-r6} */
41 .macro restore_vfp_state vfp_base
42 VFPFLDMIA \vfp_base, r6 @ Load VFP registers
43 ldm \vfp_base, {r2-r5} @ Load FPEXC, FPSCR, FPINST, FPINST2
46 tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
50 VFPFMXR FPINST2, r5, ne
52 VFPFMXR FPEXC, r2 @ FPEXC (last, in case !EN)
55 /* These are simply for the macros to work - value don't have meaning */
63 .macro push_host_regs_mode mode
71 * Store all host persistent registers on the stack.
72 * Clobbers all registers, in all modes, except r0 and r1.
75 /* Hyp regs. Only ELR_hyp (SPSR_hyp already saved) */
80 push {r4-r12} @ r0-r3 are always clobbered
85 push_host_regs_mode svc
86 push_host_regs_mode abt
87 push_host_regs_mode und
88 push_host_regs_mode irq
102 .macro pop_host_regs_mode mode
110 * Restore all host registers from the stack.
111 * Clobbers all registers, in all modes, except r0 and r1.
113 .macro restore_host_regs
124 pop_host_regs_mode irq
125 pop_host_regs_mode und
126 pop_host_regs_mode abt
127 pop_host_regs_mode svc
139 * Restore SP, LR and SPSR for a given mode. offset is the offset of
140 * this mode's registers from the VCPU base.
142 * Assumes vcpu pointer in vcpu reg
144 * Clobbers r1, r2, r3, r4.
146 .macro restore_guest_regs_mode mode, offset
147 add r1, vcpu, \offset
155 * Restore all guest registers from the vcpu struct.
157 * Assumes vcpu pointer in vcpu reg
159 * Clobbers *all* registers.
161 .macro restore_guest_regs
162 restore_guest_regs_mode svc, #VCPU_SVC_REGS
163 restore_guest_regs_mode abt, #VCPU_ABT_REGS
164 restore_guest_regs_mode und, #VCPU_UND_REGS
165 restore_guest_regs_mode irq, #VCPU_IRQ_REGS
167 add r1, vcpu, #VCPU_FIQ_REGS
179 ldr r2, [vcpu, #VCPU_PC]
180 ldr r3, [vcpu, #VCPU_CPSR]
184 @ Load user registers
185 ldr r2, [vcpu, #VCPU_USR_SP]
186 ldr r3, [vcpu, #VCPU_USR_LR]
189 add vcpu, vcpu, #(VCPU_USR_REGS)
194 * Save SP, LR and SPSR for a given mode. offset is the offset of
195 * this mode's registers from the VCPU base.
197 * Assumes vcpu pointer in vcpu reg
199 * Clobbers r2, r3, r4, r5.
201 .macro save_guest_regs_mode mode, offset
202 add r2, vcpu, \offset
210 * Save all guest registers to the vcpu struct
211 * Expects guest's r0, r1, r2 on the stack.
213 * Assumes vcpu pointer in vcpu reg
215 * Clobbers r2, r3, r4, r5.
217 .macro save_guest_regs
218 @ Store usr registers
219 add r2, vcpu, #VCPU_USR_REG(3)
221 add r2, vcpu, #VCPU_USR_REG(0)
222 pop {r3, r4, r5} @ r0, r1, r2
226 str r2, [vcpu, #VCPU_USR_SP]
227 str r3, [vcpu, #VCPU_USR_LR]
232 str r2, [vcpu, #VCPU_PC]
233 str r3, [vcpu, #VCPU_CPSR]
235 @ Store other guest registers
236 save_guest_regs_mode svc, #VCPU_SVC_REGS
237 save_guest_regs_mode abt, #VCPU_ABT_REGS
238 save_guest_regs_mode und, #VCPU_UND_REGS
239 save_guest_regs_mode irq, #VCPU_IRQ_REGS
242 /* Reads cp15 registers from hardware and stores them in memory
243 * @store_to_vcpu: If 0, registers are written in-order to the stack,
244 * otherwise to the VCPU struct pointed to by vcpup
246 * Assumes vcpu pointer in vcpu reg
250 .macro read_cp15_state store_to_vcpu
251 mrc p15, 0, r2, c1, c0, 0 @ SCTLR
252 mrc p15, 0, r3, c1, c0, 2 @ CPACR
253 mrc p15, 0, r4, c2, c0, 2 @ TTBCR
254 mrc p15, 0, r5, c3, c0, 0 @ DACR
255 mrrc p15, 0, r6, r7, c2 @ TTBR 0
256 mrrc p15, 1, r8, r9, c2 @ TTBR 1
257 mrc p15, 0, r10, c10, c2, 0 @ PRRR
258 mrc p15, 0, r11, c10, c2, 1 @ NMRR
259 mrc p15, 2, r12, c0, c0, 0 @ CSSELR
261 .if \store_to_vcpu == 0
262 push {r2-r12} @ Push CP15 registers
264 str r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
265 str r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
266 str r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
267 str r5, [vcpu, #CP15_OFFSET(c3_DACR)]
268 add r2, vcpu, #CP15_OFFSET(c2_TTBR0)
270 add r2, vcpu, #CP15_OFFSET(c2_TTBR1)
272 str r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
273 str r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
274 str r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
277 mrc p15, 0, r2, c13, c0, 1 @ CID
278 mrc p15, 0, r3, c13, c0, 2 @ TID_URW
279 mrc p15, 0, r4, c13, c0, 3 @ TID_URO
280 mrc p15, 0, r5, c13, c0, 4 @ TID_PRIV
281 mrc p15, 0, r6, c5, c0, 0 @ DFSR
282 mrc p15, 0, r7, c5, c0, 1 @ IFSR
283 mrc p15, 0, r8, c5, c1, 0 @ ADFSR
284 mrc p15, 0, r9, c5, c1, 1 @ AIFSR
285 mrc p15, 0, r10, c6, c0, 0 @ DFAR
286 mrc p15, 0, r11, c6, c0, 2 @ IFAR
287 mrc p15, 0, r12, c12, c0, 0 @ VBAR
289 .if \store_to_vcpu == 0
290 push {r2-r12} @ Push CP15 registers
292 str r2, [vcpu, #CP15_OFFSET(c13_CID)]
293 str r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
294 str r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
295 str r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
296 str r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
297 str r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
298 str r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
299 str r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
300 str r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
301 str r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
302 str r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
305 mrc p15, 0, r2, c14, c1, 0 @ CNTKCTL
306 mrrc p15, 0, r4, r5, c7 @ PAR
307 mrc p15, 0, r6, c10, c3, 0 @ AMAIR0
308 mrc p15, 0, r7, c10, c3, 1 @ AMAIR1
310 .if \store_to_vcpu == 0
313 str r2, [vcpu, #CP15_OFFSET(c14_CNTKCTL)]
314 add r12, vcpu, #CP15_OFFSET(c7_PAR)
316 str r6, [vcpu, #CP15_OFFSET(c10_AMAIR0)]
317 str r7, [vcpu, #CP15_OFFSET(c10_AMAIR1)]
322 * Reads cp15 registers from memory and writes them to hardware
323 * @read_from_vcpu: If 0, registers are read in-order from the stack,
324 * otherwise from the VCPU struct pointed to by vcpup
326 * Assumes vcpu pointer in vcpu reg
328 .macro write_cp15_state read_from_vcpu
329 .if \read_from_vcpu == 0
332 ldr r2, [vcpu, #CP15_OFFSET(c14_CNTKCTL)]
333 add r12, vcpu, #CP15_OFFSET(c7_PAR)
335 ldr r6, [vcpu, #CP15_OFFSET(c10_AMAIR0)]
336 ldr r7, [vcpu, #CP15_OFFSET(c10_AMAIR1)]
339 mcr p15, 0, r2, c14, c1, 0 @ CNTKCTL
340 mcrr p15, 0, r4, r5, c7 @ PAR
341 mcr p15, 0, r6, c10, c3, 0 @ AMAIR0
342 mcr p15, 0, r7, c10, c3, 1 @ AMAIR1
344 .if \read_from_vcpu == 0
347 ldr r2, [vcpu, #CP15_OFFSET(c13_CID)]
348 ldr r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
349 ldr r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
350 ldr r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
351 ldr r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
352 ldr r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
353 ldr r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
354 ldr r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
355 ldr r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
356 ldr r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
357 ldr r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
360 mcr p15, 0, r2, c13, c0, 1 @ CID
361 mcr p15, 0, r3, c13, c0, 2 @ TID_URW
362 mcr p15, 0, r4, c13, c0, 3 @ TID_URO
363 mcr p15, 0, r5, c13, c0, 4 @ TID_PRIV
364 mcr p15, 0, r6, c5, c0, 0 @ DFSR
365 mcr p15, 0, r7, c5, c0, 1 @ IFSR
366 mcr p15, 0, r8, c5, c1, 0 @ ADFSR
367 mcr p15, 0, r9, c5, c1, 1 @ AIFSR
368 mcr p15, 0, r10, c6, c0, 0 @ DFAR
369 mcr p15, 0, r11, c6, c0, 2 @ IFAR
370 mcr p15, 0, r12, c12, c0, 0 @ VBAR
372 .if \read_from_vcpu == 0
375 ldr r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
376 ldr r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
377 ldr r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
378 ldr r5, [vcpu, #CP15_OFFSET(c3_DACR)]
379 add r12, vcpu, #CP15_OFFSET(c2_TTBR0)
381 add r12, vcpu, #CP15_OFFSET(c2_TTBR1)
383 ldr r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
384 ldr r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
385 ldr r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
388 mcr p15, 0, r2, c1, c0, 0 @ SCTLR
389 mcr p15, 0, r3, c1, c0, 2 @ CPACR
390 mcr p15, 0, r4, c2, c0, 2 @ TTBCR
391 mcr p15, 0, r5, c3, c0, 0 @ DACR
392 mcrr p15, 0, r6, r7, c2 @ TTBR 0
393 mcrr p15, 1, r8, r9, c2 @ TTBR 1
394 mcr p15, 0, r10, c10, c2, 0 @ PRRR
395 mcr p15, 0, r11, c10, c2, 1 @ NMRR
396 mcr p15, 2, r12, c0, c0, 0 @ CSSELR
400 * Save the VGIC CPU state into memory
402 * Assumes vcpu pointer in vcpu reg
404 .macro save_vgic_state
405 /* Get VGIC VCTRL base into r2 */
406 ldr r2, [vcpu, #VCPU_KVM]
407 ldr r2, [r2, #KVM_VGIC_VCTRL]
411 /* Compute the address of struct vgic_cpu */
412 add r11, vcpu, #VCPU_VGIC_CPU
414 /* Save all interesting registers */
415 ldr r3, [r2, #GICH_HCR]
416 ldr r4, [r2, #GICH_VMCR]
417 ldr r5, [r2, #GICH_MISR]
418 ldr r6, [r2, #GICH_EISR0]
419 ldr r7, [r2, #GICH_EISR1]
420 ldr r8, [r2, #GICH_ELRSR0]
421 ldr r9, [r2, #GICH_ELRSR1]
422 ldr r10, [r2, #GICH_APR]
430 ARM_BE8(rev r10, r10 )
432 str r3, [r11, #VGIC_V2_CPU_HCR]
433 str r4, [r11, #VGIC_V2_CPU_VMCR]
434 str r5, [r11, #VGIC_V2_CPU_MISR]
435 #ifdef CONFIG_CPU_ENDIAN_BE8
436 str r6, [r11, #(VGIC_V2_CPU_EISR + 4)]
437 str r7, [r11, #VGIC_V2_CPU_EISR]
438 str r8, [r11, #(VGIC_V2_CPU_ELRSR + 4)]
439 str r9, [r11, #VGIC_V2_CPU_ELRSR]
441 str r6, [r11, #VGIC_V2_CPU_EISR]
442 str r7, [r11, #(VGIC_V2_CPU_EISR + 4)]
443 str r8, [r11, #VGIC_V2_CPU_ELRSR]
444 str r9, [r11, #(VGIC_V2_CPU_ELRSR + 4)]
446 str r10, [r11, #VGIC_V2_CPU_APR]
450 str r5, [r2, #GICH_HCR]
452 /* Save list registers */
453 add r2, r2, #GICH_LR0
454 add r3, r11, #VGIC_V2_CPU_LR
455 ldr r4, [r11, #VGIC_CPU_NR_LR]
465 * Restore the VGIC CPU state from memory
467 * Assumes vcpu pointer in vcpu reg
469 .macro restore_vgic_state
470 /* Get VGIC VCTRL base into r2 */
471 ldr r2, [vcpu, #VCPU_KVM]
472 ldr r2, [r2, #KVM_VGIC_VCTRL]
476 /* Compute the address of struct vgic_cpu */
477 add r11, vcpu, #VCPU_VGIC_CPU
479 /* We only restore a minimal set of registers */
480 ldr r3, [r11, #VGIC_V2_CPU_HCR]
481 ldr r4, [r11, #VGIC_V2_CPU_VMCR]
482 ldr r8, [r11, #VGIC_V2_CPU_APR]
487 str r3, [r2, #GICH_HCR]
488 str r4, [r2, #GICH_VMCR]
489 str r8, [r2, #GICH_APR]
491 /* Restore list registers */
492 add r2, r2, #GICH_LR0
493 add r3, r11, #VGIC_V2_CPU_LR
494 ldr r4, [r11, #VGIC_CPU_NR_LR]
503 #define CNTHCTL_PL1PCTEN (1 << 0)
504 #define CNTHCTL_PL1PCEN (1 << 1)
507 * Save the timer state onto the VCPU and allow physical timer/counter access
510 * Assumes vcpu pointer in vcpu reg
513 .macro save_timer_state
514 ldr r4, [vcpu, #VCPU_KVM]
515 ldr r2, [r4, #KVM_TIMER_ENABLED]
519 mrc p15, 0, r2, c14, c3, 1 @ CNTV_CTL
520 str r2, [vcpu, #VCPU_TIMER_CNTV_CTL]
521 bic r2, #1 @ Clear ENABLE
522 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
525 mrrc p15, 3, rr_lo_hi(r2, r3), c14 @ CNTV_CVAL
526 ldr r4, =VCPU_TIMER_CNTV_CVAL
530 @ Ensure host CNTVCT == CNTPCT
532 mcrr p15, 4, r2, r2, c14 @ CNTVOFF
535 @ Allow physical timer/counter access for the host
536 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
537 orr r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
538 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL
542 * Load the timer state from the VCPU and deny physical timer/counter access
545 * Assumes vcpu pointer in vcpu reg
548 .macro restore_timer_state
549 @ Disallow physical timer access for the guest
550 @ Physical counter access is allowed
551 mrc p15, 4, r2, c14, c1, 0 @ CNTHCTL
552 orr r2, r2, #CNTHCTL_PL1PCTEN
553 bic r2, r2, #CNTHCTL_PL1PCEN
554 mcr p15, 4, r2, c14, c1, 0 @ CNTHCTL
556 ldr r4, [vcpu, #VCPU_KVM]
557 ldr r2, [r4, #KVM_TIMER_ENABLED]
561 ldr r2, [r4, #KVM_TIMER_CNTVOFF]
562 ldr r3, [r4, #(KVM_TIMER_CNTVOFF + 4)]
563 mcrr p15, 4, rr_lo_hi(r2, r3), c14 @ CNTVOFF
565 ldr r4, =VCPU_TIMER_CNTV_CVAL
568 mcrr p15, 3, rr_lo_hi(r2, r3), c14 @ CNTV_CVAL
571 ldr r2, [vcpu, #VCPU_TIMER_CNTV_CTL]
573 mcr p15, 0, r2, c14, c3, 1 @ CNTV_CTL
580 /* Configures the HSTR (Hyp System Trap Register) on entry/return
581 * (hardware reset value is 0) */
582 .macro set_hstr operation
583 mrc p15, 4, r2, c1, c1, 3
585 .if \operation == vmentry
586 orr r2, r2, r3 @ Trap CR{15}
588 bic r2, r2, r3 @ Don't trap any CRx accesses
590 mcr p15, 4, r2, c1, c1, 3
593 /* Configures the HCPTR (Hyp Coprocessor Trap Register) on entry/return
594 * (hardware reset value is 0). Keep previous value in r2. */
595 .macro set_hcptr operation, mask
596 mrc p15, 4, r2, c1, c1, 2
598 .if \operation == vmentry
599 orr r3, r2, r3 @ Trap coproc-accesses defined in mask
601 bic r3, r2, r3 @ Don't trap defined coproc-accesses
603 mcr p15, 4, r3, c1, c1, 2
606 /* Configures the HDCR (Hyp Debug Configuration Register) on entry/return
607 * (hardware reset value is 0) */
608 .macro set_hdcr operation
609 mrc p15, 4, r2, c1, c1, 1
610 ldr r3, =(HDCR_TPM|HDCR_TPMCR)
611 .if \operation == vmentry
612 orr r2, r2, r3 @ Trap some perfmon accesses
614 bic r2, r2, r3 @ Don't trap any perfmon accesses
616 mcr p15, 4, r2, c1, c1, 1
619 /* Enable/Disable: stage-2 trans., trap interrupts, trap wfi, trap smc */
620 .macro configure_hyp_role operation
621 .if \operation == vmentry
622 ldr r2, [vcpu, #VCPU_HCR]
623 ldr r3, [vcpu, #VCPU_IRQ_LINES]
628 mcr p15, 4, r2, c1, c1, 0 @ HCR
632 mrc p15, 4, vcpu, c13, c0, 2 @ HTPIDR