1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
154 #define NoWrite ((u64)1 << 45) /* No writeback */
156 #define X2(x...) x, x
157 #define X3(x...) X2(x), x
158 #define X4(x...) X2(x), X2(x)
159 #define X5(x...) X4(x), x
160 #define X6(x...) X4(x), X2(x)
161 #define X7(x...) X4(x), X3(x)
162 #define X8(x...) X4(x), X4(x)
163 #define X16(x...) X8(x), X8(x)
165 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
166 #define FASTOP_SIZE 8
169 * fastop functions have a special calling convention:
171 * dst: [rdx]:rax (in/out)
174 * flags: rflags (in/out)
176 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
177 * different operand sizes can be reached by calculation, rather than a jump
178 * table (which would be bigger than the code).
180 * fastop functions are declared as taking a never-defined fastop parameter,
181 * so they can't be called from C directly.
190 int (*execute)(struct x86_emulate_ctxt *ctxt);
191 const struct opcode *group;
192 const struct group_dual *gdual;
193 const struct gprefix *gprefix;
194 const struct escape *esc;
195 void (*fastop)(struct fastop *fake);
197 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201 struct opcode mod012[8];
202 struct opcode mod3[8];
206 struct opcode pfx_no;
207 struct opcode pfx_66;
208 struct opcode pfx_f2;
209 struct opcode pfx_f3;
214 struct opcode high[64];
217 /* EFLAGS bit definitions. */
218 #define EFLG_ID (1<<21)
219 #define EFLG_VIP (1<<20)
220 #define EFLG_VIF (1<<19)
221 #define EFLG_AC (1<<18)
222 #define EFLG_VM (1<<17)
223 #define EFLG_RF (1<<16)
224 #define EFLG_IOPL (3<<12)
225 #define EFLG_NT (1<<14)
226 #define EFLG_OF (1<<11)
227 #define EFLG_DF (1<<10)
228 #define EFLG_IF (1<<9)
229 #define EFLG_TF (1<<8)
230 #define EFLG_SF (1<<7)
231 #define EFLG_ZF (1<<6)
232 #define EFLG_AF (1<<4)
233 #define EFLG_PF (1<<2)
234 #define EFLG_CF (1<<0)
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
237 #define EFLG_RESERVED_ONE_MASK 2
239 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
241 if (!(ctxt->regs_valid & (1 << nr))) {
242 ctxt->regs_valid |= 1 << nr;
243 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
245 return ctxt->_regs[nr];
248 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 ctxt->regs_valid |= 1 << nr;
251 ctxt->regs_dirty |= 1 << nr;
252 return &ctxt->_regs[nr];
255 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 return reg_write(ctxt, nr);
261 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
265 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
266 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
269 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
271 ctxt->regs_dirty = 0;
272 ctxt->regs_valid = 0;
276 * Instruction emulation:
277 * Most instructions are emulated directly via a fragment of inline assembly
278 * code. This allows us to save/restore EFLAGS and thus very easily pick up
279 * any modified flags.
282 #if defined(CONFIG_X86_64)
283 #define _LO32 "k" /* force 32-bit operand */
284 #define _STK "%%rsp" /* stack pointer */
285 #elif defined(__i386__)
286 #define _LO32 "" /* force 32-bit operand */
287 #define _STK "%%esp" /* stack pointer */
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
296 /* Before executing instruction: restore necessary bits in EFLAGS. */
297 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
298 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
299 "movl %"_sav",%"_LO32 _tmp"; " \
302 "movl %"_msk",%"_LO32 _tmp"; " \
303 "andl %"_LO32 _tmp",("_STK"); " \
305 "notl %"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
307 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
309 "orl %"_LO32 _tmp",("_STK"); " \
313 /* After executing instruction: write-back necessary bits in EFLAGS. */
314 #define _POST_EFLAGS(_sav, _msk, _tmp) \
315 /* _sav |= EFLAGS & _msk; */ \
318 "andl %"_msk",%"_LO32 _tmp"; " \
319 "orl %"_LO32 _tmp",%"_sav"; "
327 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
329 __asm__ __volatile__ ( \
330 _PRE_EFLAGS("0", "4", "2") \
331 _op _suffix " %"_x"3,%1; " \
332 _POST_EFLAGS("0", "4", "2") \
333 : "=m" ((ctxt)->eflags), \
334 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
336 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
340 /* Raw emulation: instruction has two explicit operands. */
341 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
343 unsigned long _tmp; \
345 switch ((ctxt)->dst.bytes) { \
347 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
350 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
353 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
360 unsigned long _tmp; \
361 switch ((ctxt)->dst.bytes) { \
363 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
366 __emulate_2op_nobyte(ctxt, _op, \
367 _wx, _wy, _lx, _ly, _qx, _qy); \
372 /* Source operand is byte-sized and may be restricted to just %cl. */
373 #define emulate_2op_SrcB(ctxt, _op) \
374 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
376 /* Source operand is byte, word, long or quad sized. */
377 #define emulate_2op_SrcV(ctxt, _op) \
378 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
380 /* Source operand is word, long or quad sized. */
381 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
382 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
384 /* Instruction has three operands and one operand is stored in ECX register */
385 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
387 unsigned long _tmp; \
388 _type _clv = (ctxt)->src2.val; \
389 _type _srcv = (ctxt)->src.val; \
390 _type _dstv = (ctxt)->dst.val; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "5", "2") \
394 _op _suffix " %4,%1 \n" \
395 _POST_EFLAGS("0", "5", "2") \
396 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
400 (ctxt)->src2.val = (unsigned long) _clv; \
401 (ctxt)->src2.val = (unsigned long) _srcv; \
402 (ctxt)->dst.val = (unsigned long) _dstv; \
405 #define emulate_2op_cl(ctxt, _op) \
407 switch ((ctxt)->dst.bytes) { \
409 __emulate_2op_cl(ctxt, _op, "w", u16); \
412 __emulate_2op_cl(ctxt, _op, "l", u32); \
415 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
420 #define __emulate_1op(ctxt, _op, _suffix) \
422 unsigned long _tmp; \
424 __asm__ __volatile__ ( \
425 _PRE_EFLAGS("0", "3", "2") \
426 _op _suffix " %1; " \
427 _POST_EFLAGS("0", "3", "2") \
428 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
430 : "i" (EFLAGS_MASK)); \
433 /* Instruction has only one explicit operand (no source operand). */
434 #define emulate_1op(ctxt, _op) \
436 switch ((ctxt)->dst.bytes) { \
437 case 1: __emulate_1op(ctxt, _op, "b"); break; \
438 case 2: __emulate_1op(ctxt, _op, "w"); break; \
439 case 4: __emulate_1op(ctxt, _op, "l"); break; \
440 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
444 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
446 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
447 #define FOP_RET "ret \n\t"
449 #define FOP_START(op) \
450 extern void em_##op(struct fastop *fake); \
451 asm(".pushsection .text, \"ax\" \n\t" \
452 ".global em_" #op " \n\t" \
459 #define FOPNOP() FOP_ALIGN FOP_RET
461 #define FOP1E(op, dst) \
462 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
464 #define FASTOP1(op) \
469 ON64(FOP1E(op##q, rax)) \
472 #define FOP2E(op, dst, src) \
473 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
475 #define FASTOP2(op) \
477 FOP2E(op##b, al, bl) \
478 FOP2E(op##w, ax, bx) \
479 FOP2E(op##l, eax, ebx) \
480 ON64(FOP2E(op##q, rax, rbx)) \
483 /* 2 operand, word only */
484 #define FASTOP2W(op) \
487 FOP2E(op##w, ax, bx) \
488 FOP2E(op##l, eax, ebx) \
489 ON64(FOP2E(op##q, rax, rbx)) \
492 /* 2 operand, src is CL */
493 #define FASTOP2CL(op) \
495 FOP2E(op##b, al, cl) \
496 FOP2E(op##w, ax, cl) \
497 FOP2E(op##l, eax, cl) \
498 ON64(FOP2E(op##q, rax, cl)) \
501 #define FOP3E(op, dst, src, src2) \
502 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
504 /* 3-operand, word-only, src2=cl */
505 #define FASTOP3WCL(op) \
508 FOP3E(op##w, ax, bx, cl) \
509 FOP3E(op##l, eax, ebx, cl) \
510 ON64(FOP3E(op##q, rax, rbx, cl)) \
513 /* Special case for SETcc - 1 instruction per cc */
514 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
535 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
537 unsigned long _tmp; \
538 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
539 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
541 __asm__ __volatile__ ( \
542 _PRE_EFLAGS("0", "5", "1") \
544 _op _suffix " %6; " \
546 _POST_EFLAGS("0", "5", "1") \
547 ".pushsection .fixup,\"ax\" \n\t" \
548 "3: movb $1, %4 \n\t" \
551 _ASM_EXTABLE(1b, 3b) \
552 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
553 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
554 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
557 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
558 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
560 switch((ctxt)->src.bytes) { \
562 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
565 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
568 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
571 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
576 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
577 enum x86_intercept intercept,
578 enum x86_intercept_stage stage)
580 struct x86_instruction_info info = {
581 .intercept = intercept,
582 .rep_prefix = ctxt->rep_prefix,
583 .modrm_mod = ctxt->modrm_mod,
584 .modrm_reg = ctxt->modrm_reg,
585 .modrm_rm = ctxt->modrm_rm,
586 .src_val = ctxt->src.val64,
587 .src_bytes = ctxt->src.bytes,
588 .dst_bytes = ctxt->dst.bytes,
589 .ad_bytes = ctxt->ad_bytes,
590 .next_rip = ctxt->eip,
593 return ctxt->ops->intercept(ctxt, &info, stage);
596 static void assign_masked(ulong *dest, ulong src, ulong mask)
598 *dest = (*dest & ~mask) | (src & mask);
601 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
603 return (1UL << (ctxt->ad_bytes << 3)) - 1;
606 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
609 struct desc_struct ss;
611 if (ctxt->mode == X86EMUL_MODE_PROT64)
613 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
614 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
617 static int stack_size(struct x86_emulate_ctxt *ctxt)
619 return (__fls(stack_mask(ctxt)) + 1) >> 3;
622 /* Access/update address held in a register, based on addressing mode. */
623 static inline unsigned long
624 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
626 if (ctxt->ad_bytes == sizeof(unsigned long))
629 return reg & ad_mask(ctxt);
632 static inline unsigned long
633 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
635 return address_mask(ctxt, reg);
638 static void masked_increment(ulong *reg, ulong mask, int inc)
640 assign_masked(reg, *reg + inc, mask);
644 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
648 if (ctxt->ad_bytes == sizeof(unsigned long))
651 mask = ad_mask(ctxt);
652 masked_increment(reg, mask, inc);
655 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
657 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
660 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
662 register_address_increment(ctxt, &ctxt->_eip, rel);
665 static u32 desc_limit_scaled(struct desc_struct *desc)
667 u32 limit = get_desc_limit(desc);
669 return desc->g ? (limit << 12) | 0xfff : limit;
672 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
674 ctxt->has_seg_override = true;
675 ctxt->seg_override = seg;
678 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
680 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
683 return ctxt->ops->get_cached_segment_base(ctxt, seg);
686 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
688 if (!ctxt->has_seg_override)
691 return ctxt->seg_override;
694 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
695 u32 error, bool valid)
697 ctxt->exception.vector = vec;
698 ctxt->exception.error_code = error;
699 ctxt->exception.error_code_valid = valid;
700 return X86EMUL_PROPAGATE_FAULT;
703 static int emulate_db(struct x86_emulate_ctxt *ctxt)
705 return emulate_exception(ctxt, DB_VECTOR, 0, false);
708 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
710 return emulate_exception(ctxt, GP_VECTOR, err, true);
713 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
715 return emulate_exception(ctxt, SS_VECTOR, err, true);
718 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
720 return emulate_exception(ctxt, UD_VECTOR, 0, false);
723 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
725 return emulate_exception(ctxt, TS_VECTOR, err, true);
728 static int emulate_de(struct x86_emulate_ctxt *ctxt)
730 return emulate_exception(ctxt, DE_VECTOR, 0, false);
733 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
735 return emulate_exception(ctxt, NM_VECTOR, 0, false);
738 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
741 struct desc_struct desc;
743 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
747 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
752 struct desc_struct desc;
754 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
755 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
759 * x86 defines three classes of vector instructions: explicitly
760 * aligned, explicitly unaligned, and the rest, which change behaviour
761 * depending on whether they're AVX encoded or not.
763 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
764 * subject to the same check.
766 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
768 if (likely(size < 16))
771 if (ctxt->d & Aligned)
773 else if (ctxt->d & Unaligned)
775 else if (ctxt->d & Avx)
781 static int __linearize(struct x86_emulate_ctxt *ctxt,
782 struct segmented_address addr,
783 unsigned size, bool write, bool fetch,
786 struct desc_struct desc;
793 la = seg_base(ctxt, addr.seg) + addr.ea;
794 switch (ctxt->mode) {
795 case X86EMUL_MODE_PROT64:
796 if (((signed long)la << 16) >> 16 != la)
797 return emulate_gp(ctxt, 0);
800 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
804 /* code segment in protected mode or read-only data segment */
805 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
806 || !(desc.type & 2)) && write)
808 /* unreadable code segment */
809 if (!fetch && (desc.type & 8) && !(desc.type & 2))
811 lim = desc_limit_scaled(&desc);
812 if ((desc.type & 8) || !(desc.type & 4)) {
813 /* expand-up segment */
814 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
817 /* expand-down segment */
818 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
820 lim = desc.d ? 0xffffffff : 0xffff;
821 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
824 cpl = ctxt->ops->cpl(ctxt);
825 if (!(desc.type & 8)) {
829 } else if ((desc.type & 8) && !(desc.type & 4)) {
830 /* nonconforming code segment */
833 } else if ((desc.type & 8) && (desc.type & 4)) {
834 /* conforming code segment */
840 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
842 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
843 return emulate_gp(ctxt, 0);
845 return X86EMUL_CONTINUE;
847 if (addr.seg == VCPU_SREG_SS)
848 return emulate_ss(ctxt, sel);
850 return emulate_gp(ctxt, sel);
853 static int linearize(struct x86_emulate_ctxt *ctxt,
854 struct segmented_address addr,
855 unsigned size, bool write,
858 return __linearize(ctxt, addr, size, write, false, linear);
862 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
863 struct segmented_address addr,
870 rc = linearize(ctxt, addr, size, false, &linear);
871 if (rc != X86EMUL_CONTINUE)
873 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
877 * Fetch the next byte of the instruction being emulated which is pointed to
878 * by ctxt->_eip, then increment ctxt->_eip.
880 * Also prefetch the remaining bytes of the instruction without crossing page
881 * boundary if they are not in fetch_cache yet.
883 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
885 struct fetch_cache *fc = &ctxt->fetch;
889 if (ctxt->_eip == fc->end) {
890 unsigned long linear;
891 struct segmented_address addr = { .seg = VCPU_SREG_CS,
893 cur_size = fc->end - fc->start;
894 size = min(15UL - cur_size,
895 PAGE_SIZE - offset_in_page(ctxt->_eip));
896 rc = __linearize(ctxt, addr, size, false, true, &linear);
897 if (unlikely(rc != X86EMUL_CONTINUE))
899 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
900 size, &ctxt->exception);
901 if (unlikely(rc != X86EMUL_CONTINUE))
905 *dest = fc->data[ctxt->_eip - fc->start];
907 return X86EMUL_CONTINUE;
910 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
911 void *dest, unsigned size)
915 /* x86 instructions are limited to 15 bytes. */
916 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
917 return X86EMUL_UNHANDLEABLE;
919 rc = do_insn_fetch_byte(ctxt, dest++);
920 if (rc != X86EMUL_CONTINUE)
923 return X86EMUL_CONTINUE;
926 /* Fetch next part of the instruction being emulated. */
927 #define insn_fetch(_type, _ctxt) \
928 ({ unsigned long _x; \
929 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
930 if (rc != X86EMUL_CONTINUE) \
935 #define insn_fetch_arr(_arr, _size, _ctxt) \
936 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
937 if (rc != X86EMUL_CONTINUE) \
942 * Given the 'reg' portion of a ModRM byte, and a register block, return a
943 * pointer into the block that addresses the relevant register.
944 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
946 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
951 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
952 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
954 p = reg_rmw(ctxt, modrm_reg);
958 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
959 struct segmented_address addr,
960 u16 *size, unsigned long *address, int op_bytes)
967 rc = segmented_read_std(ctxt, addr, size, 2);
968 if (rc != X86EMUL_CONTINUE)
971 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1010 static u8 test_cc(unsigned int condition, unsigned long flags)
1013 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1015 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1016 asm("push %[flags]; popf; call *%[fastop]"
1017 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1021 static void fetch_register_operand(struct operand *op)
1023 switch (op->bytes) {
1025 op->val = *(u8 *)op->addr.reg;
1028 op->val = *(u16 *)op->addr.reg;
1031 op->val = *(u32 *)op->addr.reg;
1034 op->val = *(u64 *)op->addr.reg;
1039 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1041 ctxt->ops->get_fpu(ctxt);
1043 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1044 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1045 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1046 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1047 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1048 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1049 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1050 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1051 #ifdef CONFIG_X86_64
1052 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1053 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1054 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1055 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1056 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1057 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1058 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1059 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1063 ctxt->ops->put_fpu(ctxt);
1066 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1069 ctxt->ops->get_fpu(ctxt);
1071 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1072 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1073 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1074 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1075 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1076 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1077 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1078 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1079 #ifdef CONFIG_X86_64
1080 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1081 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1082 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1083 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1084 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1085 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1086 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1087 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1091 ctxt->ops->put_fpu(ctxt);
1094 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1096 ctxt->ops->get_fpu(ctxt);
1098 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1099 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1100 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1101 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1102 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1103 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1104 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1105 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1108 ctxt->ops->put_fpu(ctxt);
1111 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1113 ctxt->ops->get_fpu(ctxt);
1115 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1116 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1117 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1118 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1119 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1120 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1121 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1122 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1125 ctxt->ops->put_fpu(ctxt);
1128 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1130 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1131 return emulate_nm(ctxt);
1133 ctxt->ops->get_fpu(ctxt);
1134 asm volatile("fninit");
1135 ctxt->ops->put_fpu(ctxt);
1136 return X86EMUL_CONTINUE;
1139 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1143 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1144 return emulate_nm(ctxt);
1146 ctxt->ops->get_fpu(ctxt);
1147 asm volatile("fnstcw %0": "+m"(fcw));
1148 ctxt->ops->put_fpu(ctxt);
1150 /* force 2 byte destination */
1151 ctxt->dst.bytes = 2;
1152 ctxt->dst.val = fcw;
1154 return X86EMUL_CONTINUE;
1157 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1161 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1162 return emulate_nm(ctxt);
1164 ctxt->ops->get_fpu(ctxt);
1165 asm volatile("fnstsw %0": "+m"(fsw));
1166 ctxt->ops->put_fpu(ctxt);
1168 /* force 2 byte destination */
1169 ctxt->dst.bytes = 2;
1170 ctxt->dst.val = fsw;
1172 return X86EMUL_CONTINUE;
1175 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1178 unsigned reg = ctxt->modrm_reg;
1179 int highbyte_regs = ctxt->rex_prefix == 0;
1181 if (!(ctxt->d & ModRM))
1182 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1184 if (ctxt->d & Sse) {
1188 read_sse_reg(ctxt, &op->vec_val, reg);
1191 if (ctxt->d & Mmx) {
1200 if (ctxt->d & ByteOp) {
1201 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1204 op->addr.reg = decode_register(ctxt, reg, 0);
1205 op->bytes = ctxt->op_bytes;
1207 fetch_register_operand(op);
1208 op->orig_val = op->val;
1211 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1213 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1214 ctxt->modrm_seg = VCPU_SREG_SS;
1217 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1221 int index_reg = 0, base_reg = 0, scale;
1222 int rc = X86EMUL_CONTINUE;
1225 if (ctxt->rex_prefix) {
1226 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1227 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1228 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1231 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1232 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1233 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1234 ctxt->modrm_seg = VCPU_SREG_DS;
1236 if (ctxt->modrm_mod == 3) {
1238 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1239 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1240 if (ctxt->d & Sse) {
1243 op->addr.xmm = ctxt->modrm_rm;
1244 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1247 if (ctxt->d & Mmx) {
1250 op->addr.xmm = ctxt->modrm_rm & 7;
1253 fetch_register_operand(op);
1259 if (ctxt->ad_bytes == 2) {
1260 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1261 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1262 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1263 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1265 /* 16-bit ModR/M decode. */
1266 switch (ctxt->modrm_mod) {
1268 if (ctxt->modrm_rm == 6)
1269 modrm_ea += insn_fetch(u16, ctxt);
1272 modrm_ea += insn_fetch(s8, ctxt);
1275 modrm_ea += insn_fetch(u16, ctxt);
1278 switch (ctxt->modrm_rm) {
1280 modrm_ea += bx + si;
1283 modrm_ea += bx + di;
1286 modrm_ea += bp + si;
1289 modrm_ea += bp + di;
1298 if (ctxt->modrm_mod != 0)
1305 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1306 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1307 ctxt->modrm_seg = VCPU_SREG_SS;
1308 modrm_ea = (u16)modrm_ea;
1310 /* 32/64-bit ModR/M decode. */
1311 if ((ctxt->modrm_rm & 7) == 4) {
1312 sib = insn_fetch(u8, ctxt);
1313 index_reg |= (sib >> 3) & 7;
1314 base_reg |= sib & 7;
1317 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1318 modrm_ea += insn_fetch(s32, ctxt);
1320 modrm_ea += reg_read(ctxt, base_reg);
1321 adjust_modrm_seg(ctxt, base_reg);
1324 modrm_ea += reg_read(ctxt, index_reg) << scale;
1325 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1326 if (ctxt->mode == X86EMUL_MODE_PROT64)
1327 ctxt->rip_relative = 1;
1329 base_reg = ctxt->modrm_rm;
1330 modrm_ea += reg_read(ctxt, base_reg);
1331 adjust_modrm_seg(ctxt, base_reg);
1333 switch (ctxt->modrm_mod) {
1335 if (ctxt->modrm_rm == 5)
1336 modrm_ea += insn_fetch(s32, ctxt);
1339 modrm_ea += insn_fetch(s8, ctxt);
1342 modrm_ea += insn_fetch(s32, ctxt);
1346 op->addr.mem.ea = modrm_ea;
1351 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1354 int rc = X86EMUL_CONTINUE;
1357 switch (ctxt->ad_bytes) {
1359 op->addr.mem.ea = insn_fetch(u16, ctxt);
1362 op->addr.mem.ea = insn_fetch(u32, ctxt);
1365 op->addr.mem.ea = insn_fetch(u64, ctxt);
1372 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1376 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1377 mask = ~(ctxt->dst.bytes * 8 - 1);
1379 if (ctxt->src.bytes == 2)
1380 sv = (s16)ctxt->src.val & (s16)mask;
1381 else if (ctxt->src.bytes == 4)
1382 sv = (s32)ctxt->src.val & (s32)mask;
1384 ctxt->dst.addr.mem.ea += (sv >> 3);
1387 /* only subword offset */
1388 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1391 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1392 unsigned long addr, void *dest, unsigned size)
1395 struct read_cache *mc = &ctxt->mem_read;
1397 if (mc->pos < mc->end)
1400 WARN_ON((mc->end + size) >= sizeof(mc->data));
1402 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1404 if (rc != X86EMUL_CONTINUE)
1410 memcpy(dest, mc->data + mc->pos, size);
1412 return X86EMUL_CONTINUE;
1415 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1416 struct segmented_address addr,
1423 rc = linearize(ctxt, addr, size, false, &linear);
1424 if (rc != X86EMUL_CONTINUE)
1426 return read_emulated(ctxt, linear, data, size);
1429 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1430 struct segmented_address addr,
1437 rc = linearize(ctxt, addr, size, true, &linear);
1438 if (rc != X86EMUL_CONTINUE)
1440 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1444 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1445 struct segmented_address addr,
1446 const void *orig_data, const void *data,
1452 rc = linearize(ctxt, addr, size, true, &linear);
1453 if (rc != X86EMUL_CONTINUE)
1455 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1456 size, &ctxt->exception);
1459 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1460 unsigned int size, unsigned short port,
1463 struct read_cache *rc = &ctxt->io_read;
1465 if (rc->pos == rc->end) { /* refill pio read ahead */
1466 unsigned int in_page, n;
1467 unsigned int count = ctxt->rep_prefix ?
1468 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1469 in_page = (ctxt->eflags & EFLG_DF) ?
1470 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1471 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1472 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1476 rc->pos = rc->end = 0;
1477 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1482 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1483 ctxt->dst.data = rc->data + rc->pos;
1484 ctxt->dst.type = OP_MEM_STR;
1485 ctxt->dst.count = (rc->end - rc->pos) / size;
1488 memcpy(dest, rc->data + rc->pos, size);
1494 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1495 u16 index, struct desc_struct *desc)
1500 ctxt->ops->get_idt(ctxt, &dt);
1502 if (dt.size < index * 8 + 7)
1503 return emulate_gp(ctxt, index << 3 | 0x2);
1505 addr = dt.address + index * 8;
1506 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1510 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1511 u16 selector, struct desc_ptr *dt)
1513 const struct x86_emulate_ops *ops = ctxt->ops;
1515 if (selector & 1 << 2) {
1516 struct desc_struct desc;
1519 memset (dt, 0, sizeof *dt);
1520 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1523 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1524 dt->address = get_desc_base(&desc);
1526 ops->get_gdt(ctxt, dt);
1529 /* allowed just for 8 bytes segments */
1530 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1531 u16 selector, struct desc_struct *desc,
1535 u16 index = selector >> 3;
1538 get_descriptor_table_ptr(ctxt, selector, &dt);
1540 if (dt.size < index * 8 + 7)
1541 return emulate_gp(ctxt, selector & 0xfffc);
1543 *desc_addr_p = addr = dt.address + index * 8;
1544 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1548 /* allowed just for 8 bytes segments */
1549 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1550 u16 selector, struct desc_struct *desc)
1553 u16 index = selector >> 3;
1556 get_descriptor_table_ptr(ctxt, selector, &dt);
1558 if (dt.size < index * 8 + 7)
1559 return emulate_gp(ctxt, selector & 0xfffc);
1561 addr = dt.address + index * 8;
1562 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1566 /* Does not support long mode */
1567 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1568 u16 selector, int seg)
1570 struct desc_struct seg_desc, old_desc;
1572 unsigned err_vec = GP_VECTOR;
1574 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1579 memset(&seg_desc, 0, sizeof seg_desc);
1581 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1582 || ctxt->mode == X86EMUL_MODE_REAL) {
1583 /* set real mode segment descriptor */
1584 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1585 set_desc_base(&seg_desc, selector << 4);
1590 cpl = ctxt->ops->cpl(ctxt);
1592 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1593 if ((seg == VCPU_SREG_CS
1594 || (seg == VCPU_SREG_SS
1595 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1596 || seg == VCPU_SREG_TR)
1600 /* TR should be in GDT only */
1601 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1604 if (null_selector) /* for NULL selector skip all following checks */
1607 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1608 if (ret != X86EMUL_CONTINUE)
1611 err_code = selector & 0xfffc;
1612 err_vec = GP_VECTOR;
1614 /* can't load system descriptor into segment selector */
1615 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1619 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1628 * segment is not a writable data segment or segment
1629 * selector's RPL != CPL or segment selector's RPL != CPL
1631 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1635 if (!(seg_desc.type & 8))
1638 if (seg_desc.type & 4) {
1644 if (rpl > cpl || dpl != cpl)
1647 /* CS(RPL) <- CPL */
1648 selector = (selector & 0xfffc) | cpl;
1651 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1653 old_desc = seg_desc;
1654 seg_desc.type |= 2; /* busy */
1655 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1656 sizeof(seg_desc), &ctxt->exception);
1657 if (ret != X86EMUL_CONTINUE)
1660 case VCPU_SREG_LDTR:
1661 if (seg_desc.s || seg_desc.type != 2)
1664 default: /* DS, ES, FS, or GS */
1666 * segment is not a data or readable code segment or
1667 * ((segment is a data or nonconforming code segment)
1668 * and (both RPL and CPL > DPL))
1670 if ((seg_desc.type & 0xa) == 0x8 ||
1671 (((seg_desc.type & 0xc) != 0xc) &&
1672 (rpl > dpl && cpl > dpl)))
1678 /* mark segment as accessed */
1680 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1681 if (ret != X86EMUL_CONTINUE)
1685 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1686 return X86EMUL_CONTINUE;
1688 emulate_exception(ctxt, err_vec, err_code, true);
1689 return X86EMUL_PROPAGATE_FAULT;
1692 static void write_register_operand(struct operand *op)
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1695 switch (op->bytes) {
1697 *(u8 *)op->addr.reg = (u8)op->val;
1700 *(u16 *)op->addr.reg = (u16)op->val;
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1706 *op->addr.reg = op->val;
1711 static int writeback(struct x86_emulate_ctxt *ctxt)
1715 if (ctxt->d & NoWrite)
1716 return X86EMUL_CONTINUE;
1718 switch (ctxt->dst.type) {
1720 write_register_operand(&ctxt->dst);
1723 if (ctxt->lock_prefix)
1724 rc = segmented_cmpxchg(ctxt,
1726 &ctxt->dst.orig_val,
1730 rc = segmented_write(ctxt,
1734 if (rc != X86EMUL_CONTINUE)
1738 rc = segmented_write(ctxt,
1741 ctxt->dst.bytes * ctxt->dst.count);
1742 if (rc != X86EMUL_CONTINUE)
1746 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1749 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1757 return X86EMUL_CONTINUE;
1760 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1762 struct segmented_address addr;
1764 rsp_increment(ctxt, -bytes);
1765 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1766 addr.seg = VCPU_SREG_SS;
1768 return segmented_write(ctxt, addr, data, bytes);
1771 static int em_push(struct x86_emulate_ctxt *ctxt)
1773 /* Disable writeback. */
1774 ctxt->dst.type = OP_NONE;
1775 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1778 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1779 void *dest, int len)
1782 struct segmented_address addr;
1784 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1785 addr.seg = VCPU_SREG_SS;
1786 rc = segmented_read(ctxt, addr, dest, len);
1787 if (rc != X86EMUL_CONTINUE)
1790 rsp_increment(ctxt, len);
1794 static int em_pop(struct x86_emulate_ctxt *ctxt)
1796 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1799 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1800 void *dest, int len)
1803 unsigned long val, change_mask;
1804 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1805 int cpl = ctxt->ops->cpl(ctxt);
1807 rc = emulate_pop(ctxt, &val, len);
1808 if (rc != X86EMUL_CONTINUE)
1811 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1812 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1814 switch(ctxt->mode) {
1815 case X86EMUL_MODE_PROT64:
1816 case X86EMUL_MODE_PROT32:
1817 case X86EMUL_MODE_PROT16:
1819 change_mask |= EFLG_IOPL;
1821 change_mask |= EFLG_IF;
1823 case X86EMUL_MODE_VM86:
1825 return emulate_gp(ctxt, 0);
1826 change_mask |= EFLG_IF;
1828 default: /* real mode */
1829 change_mask |= (EFLG_IOPL | EFLG_IF);
1833 *(unsigned long *)dest =
1834 (ctxt->eflags & ~change_mask) | (val & change_mask);
1839 static int em_popf(struct x86_emulate_ctxt *ctxt)
1841 ctxt->dst.type = OP_REG;
1842 ctxt->dst.addr.reg = &ctxt->eflags;
1843 ctxt->dst.bytes = ctxt->op_bytes;
1844 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1847 static int em_enter(struct x86_emulate_ctxt *ctxt)
1850 unsigned frame_size = ctxt->src.val;
1851 unsigned nesting_level = ctxt->src2.val & 31;
1855 return X86EMUL_UNHANDLEABLE;
1857 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1858 rc = push(ctxt, &rbp, stack_size(ctxt));
1859 if (rc != X86EMUL_CONTINUE)
1861 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1863 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1864 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1866 return X86EMUL_CONTINUE;
1869 static int em_leave(struct x86_emulate_ctxt *ctxt)
1871 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1873 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1876 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1878 int seg = ctxt->src2.val;
1880 ctxt->src.val = get_segment_selector(ctxt, seg);
1882 return em_push(ctxt);
1885 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1887 int seg = ctxt->src2.val;
1888 unsigned long selector;
1891 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1892 if (rc != X86EMUL_CONTINUE)
1895 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1899 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1901 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1902 int rc = X86EMUL_CONTINUE;
1903 int reg = VCPU_REGS_RAX;
1905 while (reg <= VCPU_REGS_RDI) {
1906 (reg == VCPU_REGS_RSP) ?
1907 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1910 if (rc != X86EMUL_CONTINUE)
1919 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1921 ctxt->src.val = (unsigned long)ctxt->eflags;
1922 return em_push(ctxt);
1925 static int em_popa(struct x86_emulate_ctxt *ctxt)
1927 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI;
1930 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) {
1932 rsp_increment(ctxt, ctxt->op_bytes);
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE)
1944 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1946 const struct x86_emulate_ops *ops = ctxt->ops;
1953 /* TODO: Add limit checks */
1954 ctxt->src.val = ctxt->eflags;
1956 if (rc != X86EMUL_CONTINUE)
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1963 if (rc != X86EMUL_CONTINUE)
1966 ctxt->src.val = ctxt->_eip;
1968 if (rc != X86EMUL_CONTINUE)
1971 ops->get_idt(ctxt, &dt);
1973 eip_addr = dt.address + (irq << 2);
1974 cs_addr = dt.address + (irq << 2) + 2;
1976 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1977 if (rc != X86EMUL_CONTINUE)
1980 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1981 if (rc != X86EMUL_CONTINUE)
1984 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1993 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997 invalidate_registers(ctxt);
1998 rc = __emulate_int_real(ctxt, irq);
1999 if (rc == X86EMUL_CONTINUE)
2000 writeback_registers(ctxt);
2004 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2006 switch(ctxt->mode) {
2007 case X86EMUL_MODE_REAL:
2008 return __emulate_int_real(ctxt, irq);
2009 case X86EMUL_MODE_VM86:
2010 case X86EMUL_MODE_PROT16:
2011 case X86EMUL_MODE_PROT32:
2012 case X86EMUL_MODE_PROT64:
2014 /* Protected mode interrupts unimplemented yet */
2015 return X86EMUL_UNHANDLEABLE;
2019 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2021 int rc = X86EMUL_CONTINUE;
2022 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2030 /* TODO: Add stack limit check */
2032 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2037 if (temp_eip & ~0xffff)
2038 return emulate_gp(ctxt, 0);
2040 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2042 if (rc != X86EMUL_CONTINUE)
2045 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2047 if (rc != X86EMUL_CONTINUE)
2050 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2052 if (rc != X86EMUL_CONTINUE)
2055 ctxt->_eip = temp_eip;
2058 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) {
2061 ctxt->eflags &= ~0xffff;
2062 ctxt->eflags |= temp_eflags;
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2071 static int em_iret(struct x86_emulate_ctxt *ctxt)
2073 switch(ctxt->mode) {
2074 case X86EMUL_MODE_REAL:
2075 return emulate_iret_real(ctxt);
2076 case X86EMUL_MODE_VM86:
2077 case X86EMUL_MODE_PROT16:
2078 case X86EMUL_MODE_PROT32:
2079 case X86EMUL_MODE_PROT64:
2081 /* iret from protected mode unimplemented yet */
2082 return X86EMUL_UNHANDLEABLE;
2086 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2091 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2093 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2094 if (rc != X86EMUL_CONTINUE)
2098 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2099 return X86EMUL_CONTINUE;
2102 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2106 emulate_1op_rax_rdx(ctxt, "mul", ex);
2107 return X86EMUL_CONTINUE;
2110 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2114 emulate_1op_rax_rdx(ctxt, "imul", ex);
2115 return X86EMUL_CONTINUE;
2118 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2122 emulate_1op_rax_rdx(ctxt, "div", de);
2124 return emulate_de(ctxt);
2125 return X86EMUL_CONTINUE;
2128 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2132 emulate_1op_rax_rdx(ctxt, "idiv", de);
2134 return emulate_de(ctxt);
2135 return X86EMUL_CONTINUE;
2138 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2140 int rc = X86EMUL_CONTINUE;
2142 switch (ctxt->modrm_reg) {
2143 case 2: /* call near abs */ {
2145 old_eip = ctxt->_eip;
2146 ctxt->_eip = ctxt->src.val;
2147 ctxt->src.val = old_eip;
2151 case 4: /* jmp abs */
2152 ctxt->_eip = ctxt->src.val;
2154 case 5: /* jmp far */
2155 rc = em_jmp_far(ctxt);
2164 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2166 u64 old = ctxt->dst.orig_val64;
2168 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2169 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2170 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2171 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2172 ctxt->eflags &= ~EFLG_ZF;
2174 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2175 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2177 ctxt->eflags |= EFLG_ZF;
2179 return X86EMUL_CONTINUE;
2182 static int em_ret(struct x86_emulate_ctxt *ctxt)
2184 ctxt->dst.type = OP_REG;
2185 ctxt->dst.addr.reg = &ctxt->_eip;
2186 ctxt->dst.bytes = ctxt->op_bytes;
2187 return em_pop(ctxt);
2190 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2195 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2196 if (rc != X86EMUL_CONTINUE)
2198 if (ctxt->op_bytes == 4)
2199 ctxt->_eip = (u32)ctxt->_eip;
2200 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2201 if (rc != X86EMUL_CONTINUE)
2203 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2207 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2209 /* Save real source value, then compare EAX against destination. */
2210 ctxt->src.orig_val = ctxt->src.val;
2211 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2212 fastop(ctxt, em_cmp);
2214 if (ctxt->eflags & EFLG_ZF) {
2215 /* Success: write back to memory. */
2216 ctxt->dst.val = ctxt->src.orig_val;
2218 /* Failure: write the value we saw to EAX. */
2219 ctxt->dst.type = OP_REG;
2220 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2222 return X86EMUL_CONTINUE;
2225 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2227 int seg = ctxt->src2.val;
2231 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2233 rc = load_segment_descriptor(ctxt, sel, seg);
2234 if (rc != X86EMUL_CONTINUE)
2237 ctxt->dst.val = ctxt->src.val;
2242 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2243 struct desc_struct *cs, struct desc_struct *ss)
2245 cs->l = 0; /* will be adjusted later */
2246 set_desc_base(cs, 0); /* flat segment */
2247 cs->g = 1; /* 4kb granularity */
2248 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2249 cs->type = 0x0b; /* Read, Execute, Accessed */
2251 cs->dpl = 0; /* will be adjusted later */
2256 set_desc_base(ss, 0); /* flat segment */
2257 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2258 ss->g = 1; /* 4kb granularity */
2260 ss->type = 0x03; /* Read/Write, Accessed */
2261 ss->d = 1; /* 32bit stack segment */
2268 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2270 u32 eax, ebx, ecx, edx;
2273 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2274 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2275 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2276 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2279 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2281 const struct x86_emulate_ops *ops = ctxt->ops;
2282 u32 eax, ebx, ecx, edx;
2285 * syscall should always be enabled in longmode - so only become
2286 * vendor specific (cpuid) if other modes are active...
2288 if (ctxt->mode == X86EMUL_MODE_PROT64)
2293 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2295 * Intel ("GenuineIntel")
2296 * remark: Intel CPUs only support "syscall" in 64bit
2297 * longmode. Also an 64bit guest with a
2298 * 32bit compat-app running will #UD !! While this
2299 * behaviour can be fixed (by emulating) into AMD
2300 * response - CPUs of AMD can't behave like Intel.
2302 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2303 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2304 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2307 /* AMD ("AuthenticAMD") */
2308 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2309 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2310 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2313 /* AMD ("AMDisbetter!") */
2314 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2315 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2316 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2319 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2323 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2325 const struct x86_emulate_ops *ops = ctxt->ops;
2326 struct desc_struct cs, ss;
2331 /* syscall is not available in real mode */
2332 if (ctxt->mode == X86EMUL_MODE_REAL ||
2333 ctxt->mode == X86EMUL_MODE_VM86)
2334 return emulate_ud(ctxt);
2336 if (!(em_syscall_is_enabled(ctxt)))
2337 return emulate_ud(ctxt);
2339 ops->get_msr(ctxt, MSR_EFER, &efer);
2340 setup_syscalls_segments(ctxt, &cs, &ss);
2342 if (!(efer & EFER_SCE))
2343 return emulate_ud(ctxt);
2345 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2347 cs_sel = (u16)(msr_data & 0xfffc);
2348 ss_sel = (u16)(msr_data + 8);
2350 if (efer & EFER_LMA) {
2354 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2355 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2357 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2358 if (efer & EFER_LMA) {
2359 #ifdef CONFIG_X86_64
2360 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2363 ctxt->mode == X86EMUL_MODE_PROT64 ?
2364 MSR_LSTAR : MSR_CSTAR, &msr_data);
2365 ctxt->_eip = msr_data;
2367 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2368 ctxt->eflags &= ~(msr_data | EFLG_RF);
2372 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2373 ctxt->_eip = (u32)msr_data;
2375 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2378 return X86EMUL_CONTINUE;
2381 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2383 const struct x86_emulate_ops *ops = ctxt->ops;
2384 struct desc_struct cs, ss;
2389 ops->get_msr(ctxt, MSR_EFER, &efer);
2390 /* inject #GP if in real mode */
2391 if (ctxt->mode == X86EMUL_MODE_REAL)
2392 return emulate_gp(ctxt, 0);
2395 * Not recognized on AMD in compat mode (but is recognized in legacy
2398 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2399 && !vendor_intel(ctxt))
2400 return emulate_ud(ctxt);
2402 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2403 * Therefore, we inject an #UD.
2405 if (ctxt->mode == X86EMUL_MODE_PROT64)
2406 return emulate_ud(ctxt);
2408 setup_syscalls_segments(ctxt, &cs, &ss);
2410 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2411 switch (ctxt->mode) {
2412 case X86EMUL_MODE_PROT32:
2413 if ((msr_data & 0xfffc) == 0x0)
2414 return emulate_gp(ctxt, 0);
2416 case X86EMUL_MODE_PROT64:
2417 if (msr_data == 0x0)
2418 return emulate_gp(ctxt, 0);
2424 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2425 cs_sel = (u16)msr_data;
2426 cs_sel &= ~SELECTOR_RPL_MASK;
2427 ss_sel = cs_sel + 8;
2428 ss_sel &= ~SELECTOR_RPL_MASK;
2429 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2434 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2435 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2437 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2438 ctxt->_eip = msr_data;
2440 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2441 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2443 return X86EMUL_CONTINUE;
2446 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2448 const struct x86_emulate_ops *ops = ctxt->ops;
2449 struct desc_struct cs, ss;
2452 u16 cs_sel = 0, ss_sel = 0;
2454 /* inject #GP if in real mode or Virtual 8086 mode */
2455 if (ctxt->mode == X86EMUL_MODE_REAL ||
2456 ctxt->mode == X86EMUL_MODE_VM86)
2457 return emulate_gp(ctxt, 0);
2459 setup_syscalls_segments(ctxt, &cs, &ss);
2461 if ((ctxt->rex_prefix & 0x8) != 0x0)
2462 usermode = X86EMUL_MODE_PROT64;
2464 usermode = X86EMUL_MODE_PROT32;
2468 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2470 case X86EMUL_MODE_PROT32:
2471 cs_sel = (u16)(msr_data + 16);
2472 if ((msr_data & 0xfffc) == 0x0)
2473 return emulate_gp(ctxt, 0);
2474 ss_sel = (u16)(msr_data + 24);
2476 case X86EMUL_MODE_PROT64:
2477 cs_sel = (u16)(msr_data + 32);
2478 if (msr_data == 0x0)
2479 return emulate_gp(ctxt, 0);
2480 ss_sel = cs_sel + 8;
2485 cs_sel |= SELECTOR_RPL_MASK;
2486 ss_sel |= SELECTOR_RPL_MASK;
2488 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2489 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2491 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2492 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2494 return X86EMUL_CONTINUE;
2497 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2500 if (ctxt->mode == X86EMUL_MODE_REAL)
2502 if (ctxt->mode == X86EMUL_MODE_VM86)
2504 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2505 return ctxt->ops->cpl(ctxt) > iopl;
2508 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2511 const struct x86_emulate_ops *ops = ctxt->ops;
2512 struct desc_struct tr_seg;
2515 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2516 unsigned mask = (1 << len) - 1;
2519 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2522 if (desc_limit_scaled(&tr_seg) < 103)
2524 base = get_desc_base(&tr_seg);
2525 #ifdef CONFIG_X86_64
2526 base |= ((u64)base3) << 32;
2528 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2529 if (r != X86EMUL_CONTINUE)
2531 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2533 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2534 if (r != X86EMUL_CONTINUE)
2536 if ((perm >> bit_idx) & mask)
2541 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2547 if (emulator_bad_iopl(ctxt))
2548 if (!emulator_io_port_access_allowed(ctxt, port, len))
2551 ctxt->perm_ok = true;
2556 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2557 struct tss_segment_16 *tss)
2559 tss->ip = ctxt->_eip;
2560 tss->flag = ctxt->eflags;
2561 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2562 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2563 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2564 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2565 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2566 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2567 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2568 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2570 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2571 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2572 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2573 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2574 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2577 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2578 struct tss_segment_16 *tss)
2582 ctxt->_eip = tss->ip;
2583 ctxt->eflags = tss->flag | 2;
2584 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2585 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2586 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2587 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2588 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2589 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2590 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2591 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2594 * SDM says that segment selectors are loaded before segment
2597 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2598 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2599 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2600 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2601 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2604 * Now load segment descriptors. If fault happens at this stage
2605 * it is handled in a context of new task
2607 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2608 if (ret != X86EMUL_CONTINUE)
2610 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2611 if (ret != X86EMUL_CONTINUE)
2613 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2614 if (ret != X86EMUL_CONTINUE)
2616 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2617 if (ret != X86EMUL_CONTINUE)
2619 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2620 if (ret != X86EMUL_CONTINUE)
2623 return X86EMUL_CONTINUE;
2626 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2627 u16 tss_selector, u16 old_tss_sel,
2628 ulong old_tss_base, struct desc_struct *new_desc)
2630 const struct x86_emulate_ops *ops = ctxt->ops;
2631 struct tss_segment_16 tss_seg;
2633 u32 new_tss_base = get_desc_base(new_desc);
2635 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2637 if (ret != X86EMUL_CONTINUE)
2638 /* FIXME: need to provide precise fault address */
2641 save_state_to_tss16(ctxt, &tss_seg);
2643 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2645 if (ret != X86EMUL_CONTINUE)
2646 /* FIXME: need to provide precise fault address */
2649 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2651 if (ret != X86EMUL_CONTINUE)
2652 /* FIXME: need to provide precise fault address */
2655 if (old_tss_sel != 0xffff) {
2656 tss_seg.prev_task_link = old_tss_sel;
2658 ret = ops->write_std(ctxt, new_tss_base,
2659 &tss_seg.prev_task_link,
2660 sizeof tss_seg.prev_task_link,
2662 if (ret != X86EMUL_CONTINUE)
2663 /* FIXME: need to provide precise fault address */
2667 return load_state_from_tss16(ctxt, &tss_seg);
2670 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2671 struct tss_segment_32 *tss)
2673 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2674 tss->eip = ctxt->_eip;
2675 tss->eflags = ctxt->eflags;
2676 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2677 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2678 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2679 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2680 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2681 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2682 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2683 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2685 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2686 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2687 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2688 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2689 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2690 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2691 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2694 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2695 struct tss_segment_32 *tss)
2699 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2700 return emulate_gp(ctxt, 0);
2701 ctxt->_eip = tss->eip;
2702 ctxt->eflags = tss->eflags | 2;
2704 /* General purpose registers */
2705 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2706 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2707 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2708 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2709 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2710 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2711 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2712 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2715 * SDM says that segment selectors are loaded before segment
2718 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2719 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2720 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2721 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2722 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2723 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2724 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2727 * If we're switching between Protected Mode and VM86, we need to make
2728 * sure to update the mode before loading the segment descriptors so
2729 * that the selectors are interpreted correctly.
2731 * Need to get rflags to the vcpu struct immediately because it
2732 * influences the CPL which is checked at least when loading the segment
2733 * descriptors and when pushing an error code to the new kernel stack.
2735 * TODO Introduce a separate ctxt->ops->set_cpl callback
2737 if (ctxt->eflags & X86_EFLAGS_VM)
2738 ctxt->mode = X86EMUL_MODE_VM86;
2740 ctxt->mode = X86EMUL_MODE_PROT32;
2742 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2745 * Now load segment descriptors. If fault happenes at this stage
2746 * it is handled in a context of new task
2748 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2749 if (ret != X86EMUL_CONTINUE)
2751 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2752 if (ret != X86EMUL_CONTINUE)
2754 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2755 if (ret != X86EMUL_CONTINUE)
2757 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2758 if (ret != X86EMUL_CONTINUE)
2760 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2761 if (ret != X86EMUL_CONTINUE)
2763 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2764 if (ret != X86EMUL_CONTINUE)
2766 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2767 if (ret != X86EMUL_CONTINUE)
2770 return X86EMUL_CONTINUE;
2773 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2774 u16 tss_selector, u16 old_tss_sel,
2775 ulong old_tss_base, struct desc_struct *new_desc)
2777 const struct x86_emulate_ops *ops = ctxt->ops;
2778 struct tss_segment_32 tss_seg;
2780 u32 new_tss_base = get_desc_base(new_desc);
2782 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2784 if (ret != X86EMUL_CONTINUE)
2785 /* FIXME: need to provide precise fault address */
2788 save_state_to_tss32(ctxt, &tss_seg);
2790 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2792 if (ret != X86EMUL_CONTINUE)
2793 /* FIXME: need to provide precise fault address */
2796 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2798 if (ret != X86EMUL_CONTINUE)
2799 /* FIXME: need to provide precise fault address */
2802 if (old_tss_sel != 0xffff) {
2803 tss_seg.prev_task_link = old_tss_sel;
2805 ret = ops->write_std(ctxt, new_tss_base,
2806 &tss_seg.prev_task_link,
2807 sizeof tss_seg.prev_task_link,
2809 if (ret != X86EMUL_CONTINUE)
2810 /* FIXME: need to provide precise fault address */
2814 return load_state_from_tss32(ctxt, &tss_seg);
2817 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2818 u16 tss_selector, int idt_index, int reason,
2819 bool has_error_code, u32 error_code)
2821 const struct x86_emulate_ops *ops = ctxt->ops;
2822 struct desc_struct curr_tss_desc, next_tss_desc;
2824 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2825 ulong old_tss_base =
2826 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2830 /* FIXME: old_tss_base == ~0 ? */
2832 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2833 if (ret != X86EMUL_CONTINUE)
2835 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2836 if (ret != X86EMUL_CONTINUE)
2839 /* FIXME: check that next_tss_desc is tss */
2842 * Check privileges. The three cases are task switch caused by...
2844 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2845 * 2. Exception/IRQ/iret: No check is performed
2846 * 3. jmp/call to TSS: Check against DPL of the TSS
2848 if (reason == TASK_SWITCH_GATE) {
2849 if (idt_index != -1) {
2850 /* Software interrupts */
2851 struct desc_struct task_gate_desc;
2854 ret = read_interrupt_descriptor(ctxt, idt_index,
2856 if (ret != X86EMUL_CONTINUE)
2859 dpl = task_gate_desc.dpl;
2860 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2861 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2863 } else if (reason != TASK_SWITCH_IRET) {
2864 int dpl = next_tss_desc.dpl;
2865 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2866 return emulate_gp(ctxt, tss_selector);
2870 desc_limit = desc_limit_scaled(&next_tss_desc);
2871 if (!next_tss_desc.p ||
2872 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2873 desc_limit < 0x2b)) {
2874 emulate_ts(ctxt, tss_selector & 0xfffc);
2875 return X86EMUL_PROPAGATE_FAULT;
2878 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2879 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2880 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2883 if (reason == TASK_SWITCH_IRET)
2884 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2886 /* set back link to prev task only if NT bit is set in eflags
2887 note that old_tss_sel is not used after this point */
2888 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2889 old_tss_sel = 0xffff;
2891 if (next_tss_desc.type & 8)
2892 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2893 old_tss_base, &next_tss_desc);
2895 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2896 old_tss_base, &next_tss_desc);
2897 if (ret != X86EMUL_CONTINUE)
2900 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2901 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2903 if (reason != TASK_SWITCH_IRET) {
2904 next_tss_desc.type |= (1 << 1); /* set busy flag */
2905 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2908 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2909 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2911 if (has_error_code) {
2912 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2913 ctxt->lock_prefix = 0;
2914 ctxt->src.val = (unsigned long) error_code;
2915 ret = em_push(ctxt);
2921 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2922 u16 tss_selector, int idt_index, int reason,
2923 bool has_error_code, u32 error_code)
2927 invalidate_registers(ctxt);
2928 ctxt->_eip = ctxt->eip;
2929 ctxt->dst.type = OP_NONE;
2931 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2932 has_error_code, error_code);
2934 if (rc == X86EMUL_CONTINUE) {
2935 ctxt->eip = ctxt->_eip;
2936 writeback_registers(ctxt);
2939 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2942 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2945 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2947 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2948 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2951 static int em_das(struct x86_emulate_ctxt *ctxt)
2954 bool af, cf, old_cf;
2956 cf = ctxt->eflags & X86_EFLAGS_CF;
2962 af = ctxt->eflags & X86_EFLAGS_AF;
2963 if ((al & 0x0f) > 9 || af) {
2965 cf = old_cf | (al >= 250);
2970 if (old_al > 0x99 || old_cf) {
2976 /* Set PF, ZF, SF */
2977 ctxt->src.type = OP_IMM;
2979 ctxt->src.bytes = 1;
2980 fastop(ctxt, em_or);
2981 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2983 ctxt->eflags |= X86_EFLAGS_CF;
2985 ctxt->eflags |= X86_EFLAGS_AF;
2986 return X86EMUL_CONTINUE;
2989 static int em_aad(struct x86_emulate_ctxt *ctxt)
2991 u8 al = ctxt->dst.val & 0xff;
2992 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2994 al = (al + (ah * ctxt->src.val)) & 0xff;
2996 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2998 /* Set PF, ZF, SF */
2999 ctxt->src.type = OP_IMM;
3001 ctxt->src.bytes = 1;
3002 fastop(ctxt, em_or);
3004 return X86EMUL_CONTINUE;
3007 static int em_call(struct x86_emulate_ctxt *ctxt)
3009 long rel = ctxt->src.val;
3011 ctxt->src.val = (unsigned long)ctxt->_eip;
3013 return em_push(ctxt);
3016 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3022 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3023 old_eip = ctxt->_eip;
3025 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3026 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3027 return X86EMUL_CONTINUE;
3030 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3032 ctxt->src.val = old_cs;
3034 if (rc != X86EMUL_CONTINUE)
3037 ctxt->src.val = old_eip;
3038 return em_push(ctxt);
3041 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3045 ctxt->dst.type = OP_REG;
3046 ctxt->dst.addr.reg = &ctxt->_eip;
3047 ctxt->dst.bytes = ctxt->op_bytes;
3048 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3049 if (rc != X86EMUL_CONTINUE)
3051 rsp_increment(ctxt, ctxt->src.val);
3052 return X86EMUL_CONTINUE;
3055 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3057 /* Write back the register source. */
3058 ctxt->src.val = ctxt->dst.val;
3059 write_register_operand(&ctxt->src);
3061 /* Write back the memory destination with implicit LOCK prefix. */
3062 ctxt->dst.val = ctxt->src.orig_val;
3063 ctxt->lock_prefix = 1;
3064 return X86EMUL_CONTINUE;
3067 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3069 ctxt->dst.val = ctxt->src2.val;
3070 return fastop(ctxt, em_imul);
3073 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3075 ctxt->dst.type = OP_REG;
3076 ctxt->dst.bytes = ctxt->src.bytes;
3077 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3078 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3080 return X86EMUL_CONTINUE;
3083 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3087 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3088 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3089 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3090 return X86EMUL_CONTINUE;
3093 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3097 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3098 return emulate_gp(ctxt, 0);
3099 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3100 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3101 return X86EMUL_CONTINUE;
3104 static int em_mov(struct x86_emulate_ctxt *ctxt)
3106 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3107 return X86EMUL_CONTINUE;
3110 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3112 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3113 return emulate_gp(ctxt, 0);
3115 /* Disable writeback. */
3116 ctxt->dst.type = OP_NONE;
3117 return X86EMUL_CONTINUE;
3120 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3124 if (ctxt->mode == X86EMUL_MODE_PROT64)
3125 val = ctxt->src.val & ~0ULL;
3127 val = ctxt->src.val & ~0U;
3129 /* #UD condition is already handled. */
3130 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3131 return emulate_gp(ctxt, 0);
3133 /* Disable writeback. */
3134 ctxt->dst.type = OP_NONE;
3135 return X86EMUL_CONTINUE;
3138 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3142 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3143 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3144 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3145 return emulate_gp(ctxt, 0);
3147 return X86EMUL_CONTINUE;
3150 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3154 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3155 return emulate_gp(ctxt, 0);
3157 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3158 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3159 return X86EMUL_CONTINUE;
3162 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3164 if (ctxt->modrm_reg > VCPU_SREG_GS)
3165 return emulate_ud(ctxt);
3167 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3168 return X86EMUL_CONTINUE;
3171 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3173 u16 sel = ctxt->src.val;
3175 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3176 return emulate_ud(ctxt);
3178 if (ctxt->modrm_reg == VCPU_SREG_SS)
3179 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3181 /* Disable writeback. */
3182 ctxt->dst.type = OP_NONE;
3183 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3186 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3188 u16 sel = ctxt->src.val;
3190 /* Disable writeback. */
3191 ctxt->dst.type = OP_NONE;
3192 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3195 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3197 u16 sel = ctxt->src.val;
3199 /* Disable writeback. */
3200 ctxt->dst.type = OP_NONE;
3201 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3204 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3209 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3210 if (rc == X86EMUL_CONTINUE)
3211 ctxt->ops->invlpg(ctxt, linear);
3212 /* Disable writeback. */
3213 ctxt->dst.type = OP_NONE;
3214 return X86EMUL_CONTINUE;
3217 static int em_clts(struct x86_emulate_ctxt *ctxt)
3221 cr0 = ctxt->ops->get_cr(ctxt, 0);
3223 ctxt->ops->set_cr(ctxt, 0, cr0);
3224 return X86EMUL_CONTINUE;
3227 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3231 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3232 return X86EMUL_UNHANDLEABLE;
3234 rc = ctxt->ops->fix_hypercall(ctxt);
3235 if (rc != X86EMUL_CONTINUE)
3238 /* Let the processor re-execute the fixed hypercall */
3239 ctxt->_eip = ctxt->eip;
3240 /* Disable writeback. */
3241 ctxt->dst.type = OP_NONE;
3242 return X86EMUL_CONTINUE;
3245 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3246 void (*get)(struct x86_emulate_ctxt *ctxt,
3247 struct desc_ptr *ptr))
3249 struct desc_ptr desc_ptr;
3251 if (ctxt->mode == X86EMUL_MODE_PROT64)
3253 get(ctxt, &desc_ptr);
3254 if (ctxt->op_bytes == 2) {
3256 desc_ptr.address &= 0x00ffffff;
3258 /* Disable writeback. */
3259 ctxt->dst.type = OP_NONE;
3260 return segmented_write(ctxt, ctxt->dst.addr.mem,
3261 &desc_ptr, 2 + ctxt->op_bytes);
3264 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3266 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3269 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3271 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3274 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3276 struct desc_ptr desc_ptr;
3279 if (ctxt->mode == X86EMUL_MODE_PROT64)
3281 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3282 &desc_ptr.size, &desc_ptr.address,
3284 if (rc != X86EMUL_CONTINUE)
3286 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3287 /* Disable writeback. */
3288 ctxt->dst.type = OP_NONE;
3289 return X86EMUL_CONTINUE;
3292 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3296 rc = ctxt->ops->fix_hypercall(ctxt);
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3303 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3305 struct desc_ptr desc_ptr;
3308 if (ctxt->mode == X86EMUL_MODE_PROT64)
3310 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3311 &desc_ptr.size, &desc_ptr.address,
3313 if (rc != X86EMUL_CONTINUE)
3315 ctxt->ops->set_idt(ctxt, &desc_ptr);
3316 /* Disable writeback. */
3317 ctxt->dst.type = OP_NONE;
3318 return X86EMUL_CONTINUE;
3321 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3323 ctxt->dst.bytes = 2;
3324 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3325 return X86EMUL_CONTINUE;
3328 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3330 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3331 | (ctxt->src.val & 0x0f));
3332 ctxt->dst.type = OP_NONE;
3333 return X86EMUL_CONTINUE;
3336 static int em_loop(struct x86_emulate_ctxt *ctxt)
3338 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3339 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3340 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3341 jmp_rel(ctxt, ctxt->src.val);
3343 return X86EMUL_CONTINUE;
3346 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3348 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3349 jmp_rel(ctxt, ctxt->src.val);
3351 return X86EMUL_CONTINUE;
3354 static int em_in(struct x86_emulate_ctxt *ctxt)
3356 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3358 return X86EMUL_IO_NEEDED;
3360 return X86EMUL_CONTINUE;
3363 static int em_out(struct x86_emulate_ctxt *ctxt)
3365 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3367 /* Disable writeback. */
3368 ctxt->dst.type = OP_NONE;
3369 return X86EMUL_CONTINUE;
3372 static int em_cli(struct x86_emulate_ctxt *ctxt)
3374 if (emulator_bad_iopl(ctxt))
3375 return emulate_gp(ctxt, 0);
3377 ctxt->eflags &= ~X86_EFLAGS_IF;
3378 return X86EMUL_CONTINUE;
3381 static int em_sti(struct x86_emulate_ctxt *ctxt)
3383 if (emulator_bad_iopl(ctxt))
3384 return emulate_gp(ctxt, 0);
3386 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3387 ctxt->eflags |= X86_EFLAGS_IF;
3388 return X86EMUL_CONTINUE;
3391 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3393 u32 eax, ebx, ecx, edx;
3395 eax = reg_read(ctxt, VCPU_REGS_RAX);
3396 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3397 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3398 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3399 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3400 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3401 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3402 return X86EMUL_CONTINUE;
3405 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3407 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3408 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3409 return X86EMUL_CONTINUE;
3412 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3414 switch (ctxt->op_bytes) {
3415 #ifdef CONFIG_X86_64
3417 asm("bswap %0" : "+r"(ctxt->dst.val));
3421 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3424 return X86EMUL_CONTINUE;
3427 static bool valid_cr(int nr)
3439 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3441 if (!valid_cr(ctxt->modrm_reg))
3442 return emulate_ud(ctxt);
3444 return X86EMUL_CONTINUE;
3447 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3449 u64 new_val = ctxt->src.val64;
3450 int cr = ctxt->modrm_reg;
3453 static u64 cr_reserved_bits[] = {
3454 0xffffffff00000000ULL,
3455 0, 0, 0, /* CR3 checked later */
3462 return emulate_ud(ctxt);
3464 if (new_val & cr_reserved_bits[cr])
3465 return emulate_gp(ctxt, 0);
3470 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3471 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3472 return emulate_gp(ctxt, 0);
3474 cr4 = ctxt->ops->get_cr(ctxt, 4);
3475 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3477 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3478 !(cr4 & X86_CR4_PAE))
3479 return emulate_gp(ctxt, 0);
3486 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3487 if (efer & EFER_LMA)
3488 rsvd = CR3_L_MODE_RESERVED_BITS;
3489 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3490 rsvd = CR3_PAE_RESERVED_BITS;
3491 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3492 rsvd = CR3_NONPAE_RESERVED_BITS;
3495 return emulate_gp(ctxt, 0);
3500 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3502 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3503 return emulate_gp(ctxt, 0);
3509 return X86EMUL_CONTINUE;
3512 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3516 ctxt->ops->get_dr(ctxt, 7, &dr7);
3518 /* Check if DR7.Global_Enable is set */
3519 return dr7 & (1 << 13);
3522 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3524 int dr = ctxt->modrm_reg;
3528 return emulate_ud(ctxt);
3530 cr4 = ctxt->ops->get_cr(ctxt, 4);
3531 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3532 return emulate_ud(ctxt);
3534 if (check_dr7_gd(ctxt))
3535 return emulate_db(ctxt);
3537 return X86EMUL_CONTINUE;
3540 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3542 u64 new_val = ctxt->src.val64;
3543 int dr = ctxt->modrm_reg;
3545 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3546 return emulate_gp(ctxt, 0);
3548 return check_dr_read(ctxt);
3551 static int check_svme(struct x86_emulate_ctxt *ctxt)
3555 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3557 if (!(efer & EFER_SVME))
3558 return emulate_ud(ctxt);
3560 return X86EMUL_CONTINUE;
3563 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3565 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3567 /* Valid physical address? */
3568 if (rax & 0xffff000000000000ULL)
3569 return emulate_gp(ctxt, 0);
3571 return check_svme(ctxt);
3574 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3576 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3578 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3579 return emulate_ud(ctxt);
3581 return X86EMUL_CONTINUE;
3584 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3586 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3587 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3589 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3591 return emulate_gp(ctxt, 0);
3593 return X86EMUL_CONTINUE;
3596 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3598 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3599 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3600 return emulate_gp(ctxt, 0);
3602 return X86EMUL_CONTINUE;
3605 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3607 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3608 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3609 return emulate_gp(ctxt, 0);
3611 return X86EMUL_CONTINUE;
3614 #define D(_y) { .flags = (_y) }
3615 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3616 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3617 .check_perm = (_p) }
3619 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3620 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3621 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3622 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3623 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3624 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3625 #define II(_f, _e, _i) \
3626 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3627 #define IIP(_f, _e, _i, _p) \
3628 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3629 .check_perm = (_p) }
3630 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3632 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3633 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3634 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3635 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3636 #define I2bvIP(_f, _e, _i, _p) \
3637 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3639 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3640 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3641 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3643 static const struct opcode group7_rm1[] = {
3644 DI(SrcNone | Priv, monitor),
3645 DI(SrcNone | Priv, mwait),
3649 static const struct opcode group7_rm3[] = {
3650 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3651 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3652 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3653 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3654 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3655 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3656 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3657 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3660 static const struct opcode group7_rm7[] = {
3662 DIP(SrcNone, rdtscp, check_rdtsc),
3666 static const struct opcode group1[] = {
3668 F(Lock | PageTable, em_or),
3671 F(Lock | PageTable, em_and),
3677 static const struct opcode group1A[] = {
3678 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3681 static const struct opcode group2[] = {
3682 F(DstMem | ModRM, em_rol),
3683 F(DstMem | ModRM, em_ror),
3684 F(DstMem | ModRM, em_rcl),
3685 F(DstMem | ModRM, em_rcr),
3686 F(DstMem | ModRM, em_shl),
3687 F(DstMem | ModRM, em_shr),
3688 F(DstMem | ModRM, em_shl),
3689 F(DstMem | ModRM, em_sar),
3692 static const struct opcode group3[] = {
3693 F(DstMem | SrcImm | NoWrite, em_test),
3694 F(DstMem | SrcImm | NoWrite, em_test),
3695 F(DstMem | SrcNone | Lock, em_not),
3696 F(DstMem | SrcNone | Lock, em_neg),
3697 I(SrcMem, em_mul_ex),
3698 I(SrcMem, em_imul_ex),
3699 I(SrcMem, em_div_ex),
3700 I(SrcMem, em_idiv_ex),
3703 static const struct opcode group4[] = {
3704 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3705 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3709 static const struct opcode group5[] = {
3710 F(DstMem | SrcNone | Lock, em_inc),
3711 F(DstMem | SrcNone | Lock, em_dec),
3712 I(SrcMem | Stack, em_grp45),
3713 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3714 I(SrcMem | Stack, em_grp45),
3715 I(SrcMemFAddr | ImplicitOps, em_grp45),
3716 I(SrcMem | Stack, em_grp45), N,
3719 static const struct opcode group6[] = {
3722 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3723 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3727 static const struct group_dual group7 = { {
3728 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3729 II(Mov | DstMem | Priv, em_sidt, sidt),
3730 II(SrcMem | Priv, em_lgdt, lgdt),
3731 II(SrcMem | Priv, em_lidt, lidt),
3732 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3733 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3734 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3736 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3738 N, EXT(0, group7_rm3),
3739 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3740 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3744 static const struct opcode group8[] = {
3746 F(DstMem | SrcImmByte | NoWrite, em_bt),
3747 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3748 F(DstMem | SrcImmByte | Lock, em_btr),
3749 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3752 static const struct group_dual group9 = { {
3753 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3755 N, N, N, N, N, N, N, N,
3758 static const struct opcode group11[] = {
3759 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3763 static const struct gprefix pfx_0f_6f_0f_7f = {
3764 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3767 static const struct gprefix pfx_vmovntpx = {
3768 I(0, em_mov), N, N, N,
3771 static const struct escape escape_d9 = { {
3772 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3775 N, N, N, N, N, N, N, N,
3777 N, N, N, N, N, N, N, N,
3779 N, N, N, N, N, N, N, N,
3781 N, N, N, N, N, N, N, N,
3783 N, N, N, N, N, N, N, N,
3785 N, N, N, N, N, N, N, N,
3787 N, N, N, N, N, N, N, N,
3789 N, N, N, N, N, N, N, N,
3792 static const struct escape escape_db = { {
3793 N, N, N, N, N, N, N, N,
3796 N, N, N, N, N, N, N, N,
3798 N, N, N, N, N, N, N, N,
3800 N, N, N, N, N, N, N, N,
3802 N, N, N, N, N, N, N, N,
3804 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3806 N, N, N, N, N, N, N, N,
3808 N, N, N, N, N, N, N, N,
3810 N, N, N, N, N, N, N, N,
3813 static const struct escape escape_dd = { {
3814 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3817 N, N, N, N, N, N, N, N,
3819 N, N, N, N, N, N, N, N,
3821 N, N, N, N, N, N, N, N,
3823 N, N, N, N, N, N, N, N,
3825 N, N, N, N, N, N, N, N,
3827 N, N, N, N, N, N, N, N,
3829 N, N, N, N, N, N, N, N,
3831 N, N, N, N, N, N, N, N,
3834 static const struct opcode opcode_table[256] = {
3836 F6ALU(Lock, em_add),
3837 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3838 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3840 F6ALU(Lock | PageTable, em_or),
3841 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3844 F6ALU(Lock, em_adc),
3845 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3846 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3848 F6ALU(Lock, em_sbb),
3849 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3850 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3852 F6ALU(Lock | PageTable, em_and), N, N,
3854 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3856 F6ALU(Lock, em_xor), N, N,
3858 F6ALU(NoWrite, em_cmp), N, N,
3860 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3862 X8(I(SrcReg | Stack, em_push)),
3864 X8(I(DstReg | Stack, em_pop)),
3866 I(ImplicitOps | Stack | No64, em_pusha),
3867 I(ImplicitOps | Stack | No64, em_popa),
3868 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3871 I(SrcImm | Mov | Stack, em_push),
3872 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3873 I(SrcImmByte | Mov | Stack, em_push),
3874 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3875 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3876 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3880 G(ByteOp | DstMem | SrcImm, group1),
3881 G(DstMem | SrcImm, group1),
3882 G(ByteOp | DstMem | SrcImm | No64, group1),
3883 G(DstMem | SrcImmByte, group1),
3884 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3885 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3887 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3888 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3889 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3890 D(ModRM | SrcMem | NoAccess | DstReg),
3891 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3894 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3896 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3897 I(SrcImmFAddr | No64, em_call_far), N,
3898 II(ImplicitOps | Stack, em_pushf, pushf),
3899 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3901 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3902 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3903 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3904 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3906 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3907 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3908 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3909 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3911 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3913 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3915 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3916 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3917 I(ImplicitOps | Stack, em_ret),
3918 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3919 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3920 G(ByteOp, group11), G(0, group11),
3922 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3923 N, I(ImplicitOps | Stack, em_ret_far),
3924 D(ImplicitOps), DI(SrcImmByte, intn),
3925 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3927 G(Src2One | ByteOp, group2), G(Src2One, group2),
3928 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3929 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3931 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3933 X3(I(SrcImmByte, em_loop)),
3934 I(SrcImmByte, em_jcxz),
3935 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3936 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3938 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3939 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3940 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3941 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3943 N, DI(ImplicitOps, icebp), N, N,
3944 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3945 G(ByteOp, group3), G(0, group3),
3947 D(ImplicitOps), D(ImplicitOps),
3948 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3949 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3952 static const struct opcode twobyte_table[256] = {
3954 G(0, group6), GD(0, &group7), N, N,
3955 N, I(ImplicitOps | VendorSpecific, em_syscall),
3956 II(ImplicitOps | Priv, em_clts, clts), N,
3957 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3958 N, D(ImplicitOps | ModRM), N, N,
3960 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3962 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3963 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3964 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3965 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3967 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3970 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3971 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3972 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3973 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3974 I(ImplicitOps | VendorSpecific, em_sysenter),
3975 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3977 N, N, N, N, N, N, N, N,
3979 X16(D(DstReg | SrcMem | ModRM | Mov)),
3981 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3986 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3991 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3995 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3997 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3998 II(ImplicitOps, em_cpuid, cpuid),
3999 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4000 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4001 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4003 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4004 DI(ImplicitOps, rsm),
4005 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4006 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4007 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4008 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4010 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4011 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4012 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4013 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4014 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4015 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4019 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4020 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4021 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4023 D2bv(DstMem | SrcReg | ModRM | Lock),
4024 N, D(DstMem | SrcReg | ModRM | Mov),
4025 N, N, N, GD(0, &group9),
4027 X8(I(DstReg, em_bswap)),
4029 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4031 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4033 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4050 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4054 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4060 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4061 unsigned size, bool sign_extension)
4063 int rc = X86EMUL_CONTINUE;
4067 op->addr.mem.ea = ctxt->_eip;
4068 /* NB. Immediates are sign-extended as necessary. */
4069 switch (op->bytes) {
4071 op->val = insn_fetch(s8, ctxt);
4074 op->val = insn_fetch(s16, ctxt);
4077 op->val = insn_fetch(s32, ctxt);
4080 op->val = insn_fetch(s64, ctxt);
4083 if (!sign_extension) {
4084 switch (op->bytes) {
4092 op->val &= 0xffffffff;
4100 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4103 int rc = X86EMUL_CONTINUE;
4107 decode_register_operand(ctxt, op);
4110 rc = decode_imm(ctxt, op, 1, false);
4113 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4117 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4118 fetch_bit_operand(ctxt);
4119 op->orig_val = op->val;
4122 ctxt->memop.bytes = 8;
4126 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4127 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4128 fetch_register_operand(op);
4129 op->orig_val = op->val;
4133 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4135 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4136 op->addr.mem.seg = VCPU_SREG_ES;
4143 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4144 fetch_register_operand(op);
4148 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4151 rc = decode_imm(ctxt, op, 1, true);
4158 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4161 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4164 ctxt->memop.bytes = 1;
4167 ctxt->memop.bytes = 2;
4170 ctxt->memop.bytes = 4;
4173 rc = decode_imm(ctxt, op, 2, false);
4176 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4180 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4182 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4183 op->addr.mem.seg = seg_override(ctxt);
4189 op->addr.mem.ea = ctxt->_eip;
4190 op->bytes = ctxt->op_bytes + 2;
4191 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4194 ctxt->memop.bytes = ctxt->op_bytes + 2;
4197 op->val = VCPU_SREG_ES;
4200 op->val = VCPU_SREG_CS;
4203 op->val = VCPU_SREG_SS;
4206 op->val = VCPU_SREG_DS;
4209 op->val = VCPU_SREG_FS;
4212 op->val = VCPU_SREG_GS;
4215 /* Special instructions do their own operand decoding. */
4217 op->type = OP_NONE; /* Disable writeback. */
4225 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4227 int rc = X86EMUL_CONTINUE;
4228 int mode = ctxt->mode;
4229 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4230 bool op_prefix = false;
4231 struct opcode opcode;
4233 ctxt->memop.type = OP_NONE;
4234 ctxt->memopp = NULL;
4235 ctxt->_eip = ctxt->eip;
4236 ctxt->fetch.start = ctxt->_eip;
4237 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4239 memcpy(ctxt->fetch.data, insn, insn_len);
4242 case X86EMUL_MODE_REAL:
4243 case X86EMUL_MODE_VM86:
4244 case X86EMUL_MODE_PROT16:
4245 def_op_bytes = def_ad_bytes = 2;
4247 case X86EMUL_MODE_PROT32:
4248 def_op_bytes = def_ad_bytes = 4;
4250 #ifdef CONFIG_X86_64
4251 case X86EMUL_MODE_PROT64:
4257 return EMULATION_FAILED;
4260 ctxt->op_bytes = def_op_bytes;
4261 ctxt->ad_bytes = def_ad_bytes;
4263 /* Legacy prefixes. */
4265 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4266 case 0x66: /* operand-size override */
4268 /* switch between 2/4 bytes */
4269 ctxt->op_bytes = def_op_bytes ^ 6;
4271 case 0x67: /* address-size override */
4272 if (mode == X86EMUL_MODE_PROT64)
4273 /* switch between 4/8 bytes */
4274 ctxt->ad_bytes = def_ad_bytes ^ 12;
4276 /* switch between 2/4 bytes */
4277 ctxt->ad_bytes = def_ad_bytes ^ 6;
4279 case 0x26: /* ES override */
4280 case 0x2e: /* CS override */
4281 case 0x36: /* SS override */
4282 case 0x3e: /* DS override */
4283 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4285 case 0x64: /* FS override */
4286 case 0x65: /* GS override */
4287 set_seg_override(ctxt, ctxt->b & 7);
4289 case 0x40 ... 0x4f: /* REX */
4290 if (mode != X86EMUL_MODE_PROT64)
4292 ctxt->rex_prefix = ctxt->b;
4294 case 0xf0: /* LOCK */
4295 ctxt->lock_prefix = 1;
4297 case 0xf2: /* REPNE/REPNZ */
4298 case 0xf3: /* REP/REPE/REPZ */
4299 ctxt->rep_prefix = ctxt->b;
4305 /* Any legacy prefix after a REX prefix nullifies its effect. */
4307 ctxt->rex_prefix = 0;
4313 if (ctxt->rex_prefix & 8)
4314 ctxt->op_bytes = 8; /* REX.W */
4316 /* Opcode byte(s). */
4317 opcode = opcode_table[ctxt->b];
4318 /* Two-byte opcode? */
4319 if (ctxt->b == 0x0f) {
4321 ctxt->b = insn_fetch(u8, ctxt);
4322 opcode = twobyte_table[ctxt->b];
4324 ctxt->d = opcode.flags;
4326 if (ctxt->d & ModRM)
4327 ctxt->modrm = insn_fetch(u8, ctxt);
4329 while (ctxt->d & GroupMask) {
4330 switch (ctxt->d & GroupMask) {
4332 goffset = (ctxt->modrm >> 3) & 7;
4333 opcode = opcode.u.group[goffset];
4336 goffset = (ctxt->modrm >> 3) & 7;
4337 if ((ctxt->modrm >> 6) == 3)
4338 opcode = opcode.u.gdual->mod3[goffset];
4340 opcode = opcode.u.gdual->mod012[goffset];
4343 goffset = ctxt->modrm & 7;
4344 opcode = opcode.u.group[goffset];
4347 if (ctxt->rep_prefix && op_prefix)
4348 return EMULATION_FAILED;
4349 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4350 switch (simd_prefix) {
4351 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4352 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4353 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4354 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4358 if (ctxt->modrm > 0xbf)
4359 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4361 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4364 return EMULATION_FAILED;
4367 ctxt->d &= ~(u64)GroupMask;
4368 ctxt->d |= opcode.flags;
4371 ctxt->execute = opcode.u.execute;
4372 ctxt->check_perm = opcode.check_perm;
4373 ctxt->intercept = opcode.intercept;
4376 if (ctxt->d == 0 || (ctxt->d & Undefined))
4377 return EMULATION_FAILED;
4379 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4380 return EMULATION_FAILED;
4382 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4385 if (ctxt->d & Op3264) {
4386 if (mode == X86EMUL_MODE_PROT64)
4393 ctxt->op_bytes = 16;
4394 else if (ctxt->d & Mmx)
4397 /* ModRM and SIB bytes. */
4398 if (ctxt->d & ModRM) {
4399 rc = decode_modrm(ctxt, &ctxt->memop);
4400 if (!ctxt->has_seg_override)
4401 set_seg_override(ctxt, ctxt->modrm_seg);
4402 } else if (ctxt->d & MemAbs)
4403 rc = decode_abs(ctxt, &ctxt->memop);
4404 if (rc != X86EMUL_CONTINUE)
4407 if (!ctxt->has_seg_override)
4408 set_seg_override(ctxt, VCPU_SREG_DS);
4410 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4412 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4413 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4416 * Decode and fetch the source operand: register, memory
4419 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4420 if (rc != X86EMUL_CONTINUE)
4424 * Decode and fetch the second source operand: register, memory
4427 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4428 if (rc != X86EMUL_CONTINUE)
4431 /* Decode and fetch the destination operand: register or memory. */
4432 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4435 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4436 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4438 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4441 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4443 return ctxt->d & PageTable;
4446 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4448 /* The second termination condition only applies for REPE
4449 * and REPNE. Test if the repeat string operation prefix is
4450 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4451 * corresponding termination condition according to:
4452 * - if REPE/REPZ and ZF = 0 then done
4453 * - if REPNE/REPNZ and ZF = 1 then done
4455 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4456 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4457 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4458 ((ctxt->eflags & EFLG_ZF) == 0))
4459 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4460 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4466 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4470 ctxt->ops->get_fpu(ctxt);
4471 asm volatile("1: fwait \n\t"
4473 ".pushsection .fixup,\"ax\" \n\t"
4475 "movb $1, %[fault] \n\t"
4478 _ASM_EXTABLE(1b, 3b)
4479 : [fault]"+qm"(fault));
4480 ctxt->ops->put_fpu(ctxt);
4482 if (unlikely(fault))
4483 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4485 return X86EMUL_CONTINUE;
4488 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4491 if (op->type == OP_MM)
4492 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4495 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4497 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4498 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4499 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4500 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4501 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4502 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4503 return X86EMUL_CONTINUE;
4506 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4508 const struct x86_emulate_ops *ops = ctxt->ops;
4509 int rc = X86EMUL_CONTINUE;
4510 int saved_dst_type = ctxt->dst.type;
4512 ctxt->mem_read.pos = 0;
4514 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4515 rc = emulate_ud(ctxt);
4519 /* LOCK prefix is allowed only with some instructions */
4520 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4521 rc = emulate_ud(ctxt);
4525 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4526 rc = emulate_ud(ctxt);
4530 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4531 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4532 rc = emulate_ud(ctxt);
4536 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4537 rc = emulate_nm(ctxt);
4541 if (ctxt->d & Mmx) {
4542 rc = flush_pending_x87_faults(ctxt);
4543 if (rc != X86EMUL_CONTINUE)
4546 * Now that we know the fpu is exception safe, we can fetch
4549 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4550 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4551 if (!(ctxt->d & Mov))
4552 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4555 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4556 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4557 X86_ICPT_PRE_EXCEPT);
4558 if (rc != X86EMUL_CONTINUE)
4562 /* Privileged instruction can be executed only in CPL=0 */
4563 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4564 rc = emulate_gp(ctxt, 0);
4568 /* Instruction can only be executed in protected mode */
4569 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4570 rc = emulate_ud(ctxt);
4574 /* Do instruction specific permission checks */
4575 if (ctxt->check_perm) {
4576 rc = ctxt->check_perm(ctxt);
4577 if (rc != X86EMUL_CONTINUE)
4581 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4582 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4583 X86_ICPT_POST_EXCEPT);
4584 if (rc != X86EMUL_CONTINUE)
4588 if (ctxt->rep_prefix && (ctxt->d & String)) {
4589 /* All REP prefixes have the same first termination condition */
4590 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4591 ctxt->eip = ctxt->_eip;
4596 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4597 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4598 ctxt->src.valptr, ctxt->src.bytes);
4599 if (rc != X86EMUL_CONTINUE)
4601 ctxt->src.orig_val64 = ctxt->src.val64;
4604 if (ctxt->src2.type == OP_MEM) {
4605 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4606 &ctxt->src2.val, ctxt->src2.bytes);
4607 if (rc != X86EMUL_CONTINUE)
4611 if ((ctxt->d & DstMask) == ImplicitOps)
4615 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4616 /* optimisation - avoid slow emulated read if Mov */
4617 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4618 &ctxt->dst.val, ctxt->dst.bytes);
4619 if (rc != X86EMUL_CONTINUE)
4622 ctxt->dst.orig_val = ctxt->dst.val;
4626 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4627 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4628 X86_ICPT_POST_MEMACCESS);
4629 if (rc != X86EMUL_CONTINUE)
4633 if (ctxt->execute) {
4634 if (ctxt->d & Fastop) {
4635 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4636 rc = fastop(ctxt, fop);
4637 if (rc != X86EMUL_CONTINUE)
4641 rc = ctxt->execute(ctxt);
4642 if (rc != X86EMUL_CONTINUE)
4651 case 0x63: /* movsxd */
4652 if (ctxt->mode != X86EMUL_MODE_PROT64)
4653 goto cannot_emulate;
4654 ctxt->dst.val = (s32) ctxt->src.val;
4656 case 0x70 ... 0x7f: /* jcc (short) */
4657 if (test_cc(ctxt->b, ctxt->eflags))
4658 jmp_rel(ctxt, ctxt->src.val);
4660 case 0x8d: /* lea r16/r32, m */
4661 ctxt->dst.val = ctxt->src.addr.mem.ea;
4663 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4664 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4668 case 0x98: /* cbw/cwde/cdqe */
4669 switch (ctxt->op_bytes) {
4670 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4671 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4672 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4675 case 0xcc: /* int3 */
4676 rc = emulate_int(ctxt, 3);
4678 case 0xcd: /* int n */
4679 rc = emulate_int(ctxt, ctxt->src.val);
4681 case 0xce: /* into */
4682 if (ctxt->eflags & EFLG_OF)
4683 rc = emulate_int(ctxt, 4);
4685 case 0xe9: /* jmp rel */
4686 case 0xeb: /* jmp rel short */
4687 jmp_rel(ctxt, ctxt->src.val);
4688 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4690 case 0xf4: /* hlt */
4691 ctxt->ops->halt(ctxt);
4693 case 0xf5: /* cmc */
4694 /* complement carry flag from eflags reg */
4695 ctxt->eflags ^= EFLG_CF;
4697 case 0xf8: /* clc */
4698 ctxt->eflags &= ~EFLG_CF;
4700 case 0xf9: /* stc */
4701 ctxt->eflags |= EFLG_CF;
4703 case 0xfc: /* cld */
4704 ctxt->eflags &= ~EFLG_DF;
4706 case 0xfd: /* std */
4707 ctxt->eflags |= EFLG_DF;
4710 goto cannot_emulate;
4713 if (rc != X86EMUL_CONTINUE)
4717 rc = writeback(ctxt);
4718 if (rc != X86EMUL_CONTINUE)
4722 * restore dst type in case the decoding will be reused
4723 * (happens for string instruction )
4725 ctxt->dst.type = saved_dst_type;
4727 if ((ctxt->d & SrcMask) == SrcSI)
4728 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4730 if ((ctxt->d & DstMask) == DstDI)
4731 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4733 if (ctxt->rep_prefix && (ctxt->d & String)) {
4735 struct read_cache *r = &ctxt->io_read;
4736 if ((ctxt->d & SrcMask) == SrcSI)
4737 count = ctxt->src.count;
4739 count = ctxt->dst.count;
4740 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4743 if (!string_insn_completed(ctxt)) {
4745 * Re-enter guest when pio read ahead buffer is empty
4746 * or, if it is not used, after each 1024 iteration.
4748 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4749 (r->end == 0 || r->end != r->pos)) {
4751 * Reset read cache. Usually happens before
4752 * decode, but since instruction is restarted
4753 * we have to do it here.
4755 ctxt->mem_read.end = 0;
4756 writeback_registers(ctxt);
4757 return EMULATION_RESTART;
4759 goto done; /* skip rip writeback */
4763 ctxt->eip = ctxt->_eip;
4766 if (rc == X86EMUL_PROPAGATE_FAULT)
4767 ctxt->have_exception = true;
4768 if (rc == X86EMUL_INTERCEPTED)
4769 return EMULATION_INTERCEPTED;
4771 if (rc == X86EMUL_CONTINUE)
4772 writeback_registers(ctxt);
4774 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4778 case 0x09: /* wbinvd */
4779 (ctxt->ops->wbinvd)(ctxt);
4781 case 0x08: /* invd */
4782 case 0x0d: /* GrpP (prefetch) */
4783 case 0x18: /* Grp16 (prefetch/nop) */
4785 case 0x20: /* mov cr, reg */
4786 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4788 case 0x21: /* mov from dr to reg */
4789 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4791 case 0x40 ... 0x4f: /* cmov */
4792 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4793 if (!test_cc(ctxt->b, ctxt->eflags))
4794 ctxt->dst.type = OP_NONE; /* no writeback */
4796 case 0x80 ... 0x8f: /* jnz rel, etc*/
4797 if (test_cc(ctxt->b, ctxt->eflags))
4798 jmp_rel(ctxt, ctxt->src.val);
4800 case 0x90 ... 0x9f: /* setcc r/m8 */
4801 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4803 case 0xae: /* clflush */
4805 case 0xb6 ... 0xb7: /* movzx */
4806 ctxt->dst.bytes = ctxt->op_bytes;
4807 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4808 : (u16) ctxt->src.val;
4810 case 0xbe ... 0xbf: /* movsx */
4811 ctxt->dst.bytes = ctxt->op_bytes;
4812 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4813 (s16) ctxt->src.val;
4815 case 0xc0 ... 0xc1: /* xadd */
4816 fastop(ctxt, em_add);
4817 /* Write back the register source. */
4818 ctxt->src.val = ctxt->dst.orig_val;
4819 write_register_operand(&ctxt->src);
4821 case 0xc3: /* movnti */
4822 ctxt->dst.bytes = ctxt->op_bytes;
4823 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4824 (u64) ctxt->src.val;
4827 goto cannot_emulate;
4830 if (rc != X86EMUL_CONTINUE)
4836 return EMULATION_FAILED;
4839 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4841 invalidate_registers(ctxt);
4844 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4846 writeback_registers(ctxt);