1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpBits 5 /* Width of operand field */
66 #define OpMask ((1ull << OpBits) - 1)
69 * Opcode effective-address decode tables.
70 * Note that we only emulate instructions that have at least one memory
71 * operand (excluding implicit stack references). We assume that stack
72 * references and instruction fetches will never occur in special memory
73 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
77 /* Operand sizes: 8-bit operands or specified/overridden size. */
78 #define ByteOp (1<<0) /* 8-bit operands. */
79 /* Destination operand type. */
81 #define ImplicitOps (OpImplicit << DstShift)
82 #define DstReg (OpReg << DstShift)
83 #define DstMem (OpMem << DstShift)
84 #define DstAcc (OpAcc << DstShift)
85 #define DstDI (OpDI << DstShift)
86 #define DstMem64 (OpMem64 << DstShift)
87 #define DstImmUByte (OpImmUByte << DstShift)
88 #define DstDX (OpDX << DstShift)
89 #define DstMask (OpMask << DstShift)
90 /* Source operand type. */
92 #define SrcNone (OpNone << SrcShift)
93 #define SrcReg (OpReg << SrcShift)
94 #define SrcMem (OpMem << SrcShift)
95 #define SrcMem16 (OpMem16 << SrcShift)
96 #define SrcMem32 (OpMem32 << SrcShift)
97 #define SrcImm (OpImm << SrcShift)
98 #define SrcImmByte (OpImmByte << SrcShift)
99 #define SrcOne (OpOne << SrcShift)
100 #define SrcImmUByte (OpImmUByte << SrcShift)
101 #define SrcImmU (OpImmU << SrcShift)
102 #define SrcSI (OpSI << SrcShift)
103 #define SrcXLat (OpXLat << SrcShift)
104 #define SrcImmFAddr (OpImmFAddr << SrcShift)
105 #define SrcMemFAddr (OpMemFAddr << SrcShift)
106 #define SrcAcc (OpAcc << SrcShift)
107 #define SrcImmU16 (OpImmU16 << SrcShift)
108 #define SrcImm64 (OpImm64 << SrcShift)
109 #define SrcDX (OpDX << SrcShift)
110 #define SrcMem8 (OpMem8 << SrcShift)
111 #define SrcMask (OpMask << SrcShift)
112 #define BitOp (1<<11)
113 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
114 #define String (1<<13) /* String instruction (rep capable) */
115 #define Stack (1<<14) /* Stack instruction (push/pop) */
116 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
117 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
118 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
119 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
120 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
121 #define Escape (5<<15) /* Escape to coprocessor instruction */
122 #define Sse (1<<18) /* SSE Vector instruction */
123 /* Generic ModRM decode. */
124 #define ModRM (1<<19)
125 /* Destination is only written; never read. */
128 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
129 #define VendorSpecific (1<<22) /* Vendor specific instruction */
130 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132 #define Undefined (1<<25) /* No Such Instruction */
133 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
134 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
136 #define PageTable (1 << 29) /* instruction used to write page table */
137 #define NotImpl (1 << 30) /* instruction is not implemented */
138 /* Source 2 operand type */
139 #define Src2Shift (31)
140 #define Src2None (OpNone << Src2Shift)
141 #define Src2CL (OpCL << Src2Shift)
142 #define Src2ImmByte (OpImmByte << Src2Shift)
143 #define Src2One (OpOne << Src2Shift)
144 #define Src2Imm (OpImm << Src2Shift)
145 #define Src2ES (OpES << Src2Shift)
146 #define Src2CS (OpCS << Src2Shift)
147 #define Src2SS (OpSS << Src2Shift)
148 #define Src2DS (OpDS << Src2Shift)
149 #define Src2FS (OpFS << Src2Shift)
150 #define Src2GS (OpGS << Src2Shift)
151 #define Src2Mask (OpMask << Src2Shift)
152 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
153 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
154 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
155 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
156 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
157 #define NoWrite ((u64)1 << 45) /* No writeback */
159 #define X2(x...) x, x
160 #define X3(x...) X2(x), x
161 #define X4(x...) X2(x), X2(x)
162 #define X5(x...) X4(x), x
163 #define X6(x...) X4(x), X2(x)
164 #define X7(x...) X4(x), X3(x)
165 #define X8(x...) X4(x), X4(x)
166 #define X16(x...) X8(x), X8(x)
168 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
169 #define FASTOP_SIZE 8
172 * fastop functions have a special calling convention:
174 * dst: [rdx]:rax (in/out)
177 * flags: rflags (in/out)
179 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
180 * different operand sizes can be reached by calculation, rather than a jump
181 * table (which would be bigger than the code).
183 * fastop functions are declared as taking a never-defined fastop parameter,
184 * so they can't be called from C directly.
193 int (*execute)(struct x86_emulate_ctxt *ctxt);
194 const struct opcode *group;
195 const struct group_dual *gdual;
196 const struct gprefix *gprefix;
197 const struct escape *esc;
198 void (*fastop)(struct fastop *fake);
200 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
204 struct opcode mod012[8];
205 struct opcode mod3[8];
209 struct opcode pfx_no;
210 struct opcode pfx_66;
211 struct opcode pfx_f2;
212 struct opcode pfx_f3;
217 struct opcode high[64];
220 /* EFLAGS bit definitions. */
221 #define EFLG_ID (1<<21)
222 #define EFLG_VIP (1<<20)
223 #define EFLG_VIF (1<<19)
224 #define EFLG_AC (1<<18)
225 #define EFLG_VM (1<<17)
226 #define EFLG_RF (1<<16)
227 #define EFLG_IOPL (3<<12)
228 #define EFLG_NT (1<<14)
229 #define EFLG_OF (1<<11)
230 #define EFLG_DF (1<<10)
231 #define EFLG_IF (1<<9)
232 #define EFLG_TF (1<<8)
233 #define EFLG_SF (1<<7)
234 #define EFLG_ZF (1<<6)
235 #define EFLG_AF (1<<4)
236 #define EFLG_PF (1<<2)
237 #define EFLG_CF (1<<0)
239 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
240 #define EFLG_RESERVED_ONE_MASK 2
242 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
244 if (!(ctxt->regs_valid & (1 << nr))) {
245 ctxt->regs_valid |= 1 << nr;
246 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
248 return ctxt->_regs[nr];
251 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 ctxt->regs_valid |= 1 << nr;
254 ctxt->regs_dirty |= 1 << nr;
255 return &ctxt->_regs[nr];
258 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
261 return reg_write(ctxt, nr);
264 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
268 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
269 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
272 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
274 ctxt->regs_dirty = 0;
275 ctxt->regs_valid = 0;
279 * Instruction emulation:
280 * Most instructions are emulated directly via a fragment of inline assembly
281 * code. This allows us to save/restore EFLAGS and thus very easily pick up
282 * any modified flags.
285 #if defined(CONFIG_X86_64)
286 #define _LO32 "k" /* force 32-bit operand */
287 #define _STK "%%rsp" /* stack pointer */
288 #elif defined(__i386__)
289 #define _LO32 "" /* force 32-bit operand */
290 #define _STK "%%esp" /* stack pointer */
294 * These EFLAGS bits are restored from saved value during emulation, and
295 * any changes are written back to the saved value after emulation.
297 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 /* Before executing instruction: restore necessary bits in EFLAGS. */
300 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
301 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
302 "movl %"_sav",%"_LO32 _tmp"; " \
305 "movl %"_msk",%"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
308 "notl %"_LO32 _tmp"; " \
309 "andl %"_LO32 _tmp",("_STK"); " \
310 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
312 "orl %"_LO32 _tmp",("_STK"); " \
316 /* After executing instruction: write-back necessary bits in EFLAGS. */
317 #define _POST_EFLAGS(_sav, _msk, _tmp) \
318 /* _sav |= EFLAGS & _msk; */ \
321 "andl %"_msk",%"_LO32 _tmp"; " \
322 "orl %"_LO32 _tmp",%"_sav"; "
330 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "2") \
334 _op _suffix " %"_x"3,%1; " \
335 _POST_EFLAGS("0", "4", "2") \
336 : "=m" ((ctxt)->eflags), \
337 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
339 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
343 /* Raw emulation: instruction has two explicit operands. */
344 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
346 unsigned long _tmp; \
348 switch ((ctxt)->dst.bytes) { \
350 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
353 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
356 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
361 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
363 unsigned long _tmp; \
364 switch ((ctxt)->dst.bytes) { \
366 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
369 __emulate_2op_nobyte(ctxt, _op, \
370 _wx, _wy, _lx, _ly, _qx, _qy); \
375 /* Source operand is byte-sized and may be restricted to just %cl. */
376 #define emulate_2op_SrcB(ctxt, _op) \
377 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
379 /* Source operand is byte, word, long or quad sized. */
380 #define emulate_2op_SrcV(ctxt, _op) \
381 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
383 /* Source operand is word, long or quad sized. */
384 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
385 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
387 /* Instruction has three operands and one operand is stored in ECX register */
388 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
390 unsigned long _tmp; \
391 _type _clv = (ctxt)->src2.val; \
392 _type _srcv = (ctxt)->src.val; \
393 _type _dstv = (ctxt)->dst.val; \
395 __asm__ __volatile__ ( \
396 _PRE_EFLAGS("0", "5", "2") \
397 _op _suffix " %4,%1 \n" \
398 _POST_EFLAGS("0", "5", "2") \
399 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
400 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
403 (ctxt)->src2.val = (unsigned long) _clv; \
404 (ctxt)->src2.val = (unsigned long) _srcv; \
405 (ctxt)->dst.val = (unsigned long) _dstv; \
408 #define emulate_2op_cl(ctxt, _op) \
410 switch ((ctxt)->dst.bytes) { \
412 __emulate_2op_cl(ctxt, _op, "w", u16); \
415 __emulate_2op_cl(ctxt, _op, "l", u32); \
418 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
423 #define __emulate_1op(ctxt, _op, _suffix) \
425 unsigned long _tmp; \
427 __asm__ __volatile__ ( \
428 _PRE_EFLAGS("0", "3", "2") \
429 _op _suffix " %1; " \
430 _POST_EFLAGS("0", "3", "2") \
431 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
433 : "i" (EFLAGS_MASK)); \
436 /* Instruction has only one explicit operand (no source operand). */
437 #define emulate_1op(ctxt, _op) \
439 switch ((ctxt)->dst.bytes) { \
440 case 1: __emulate_1op(ctxt, _op, "b"); break; \
441 case 2: __emulate_1op(ctxt, _op, "w"); break; \
442 case 4: __emulate_1op(ctxt, _op, "l"); break; \
443 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
447 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
449 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
450 #define FOP_RET "ret \n\t"
452 #define FOP_START(op) \
453 extern void em_##op(struct fastop *fake); \
454 asm(".pushsection .text, \"ax\" \n\t" \
455 ".global em_" #op " \n\t" \
462 #define FOPNOP() FOP_ALIGN FOP_RET
464 #define FOP1E(op, dst) \
465 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
467 #define FASTOP1(op) \
472 ON64(FOP1E(op##q, rax)) \
475 #define FOP2E(op, dst, src) \
476 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
478 #define FASTOP2(op) \
480 FOP2E(op##b, al, bl) \
481 FOP2E(op##w, ax, bx) \
482 FOP2E(op##l, eax, ebx) \
483 ON64(FOP2E(op##q, rax, rbx)) \
486 /* 2 operand, word only */
487 #define FASTOP2W(op) \
490 FOP2E(op##w, ax, bx) \
491 FOP2E(op##l, eax, ebx) \
492 ON64(FOP2E(op##q, rax, rbx)) \
495 /* 2 operand, src is CL */
496 #define FASTOP2CL(op) \
498 FOP2E(op##b, al, cl) \
499 FOP2E(op##w, ax, cl) \
500 FOP2E(op##l, eax, cl) \
501 ON64(FOP2E(op##q, rax, cl)) \
504 #define FOP3E(op, dst, src, src2) \
505 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
507 /* 3-operand, word-only, src2=cl */
508 #define FASTOP3WCL(op) \
511 FOP3E(op##w, ax, bx, cl) \
512 FOP3E(op##l, eax, ebx, cl) \
513 ON64(FOP3E(op##q, rax, rbx, cl)) \
516 /* Special case for SETcc - 1 instruction per cc */
517 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
538 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
541 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
543 unsigned long _tmp; \
544 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
545 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
547 __asm__ __volatile__ ( \
548 _PRE_EFLAGS("0", "5", "1") \
550 _op _suffix " %6; " \
552 _POST_EFLAGS("0", "5", "1") \
553 ".pushsection .fixup,\"ax\" \n\t" \
554 "3: movb $1, %4 \n\t" \
557 _ASM_EXTABLE(1b, 3b) \
558 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
559 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
560 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
563 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
564 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
566 switch((ctxt)->src.bytes) { \
568 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
571 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
574 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
577 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
582 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
583 enum x86_intercept intercept,
584 enum x86_intercept_stage stage)
586 struct x86_instruction_info info = {
587 .intercept = intercept,
588 .rep_prefix = ctxt->rep_prefix,
589 .modrm_mod = ctxt->modrm_mod,
590 .modrm_reg = ctxt->modrm_reg,
591 .modrm_rm = ctxt->modrm_rm,
592 .src_val = ctxt->src.val64,
593 .src_bytes = ctxt->src.bytes,
594 .dst_bytes = ctxt->dst.bytes,
595 .ad_bytes = ctxt->ad_bytes,
596 .next_rip = ctxt->eip,
599 return ctxt->ops->intercept(ctxt, &info, stage);
602 static void assign_masked(ulong *dest, ulong src, ulong mask)
604 *dest = (*dest & ~mask) | (src & mask);
607 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
609 return (1UL << (ctxt->ad_bytes << 3)) - 1;
612 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
615 struct desc_struct ss;
617 if (ctxt->mode == X86EMUL_MODE_PROT64)
619 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
620 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
623 static int stack_size(struct x86_emulate_ctxt *ctxt)
625 return (__fls(stack_mask(ctxt)) + 1) >> 3;
628 /* Access/update address held in a register, based on addressing mode. */
629 static inline unsigned long
630 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
632 if (ctxt->ad_bytes == sizeof(unsigned long))
635 return reg & ad_mask(ctxt);
638 static inline unsigned long
639 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
641 return address_mask(ctxt, reg);
644 static void masked_increment(ulong *reg, ulong mask, int inc)
646 assign_masked(reg, *reg + inc, mask);
650 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
654 if (ctxt->ad_bytes == sizeof(unsigned long))
657 mask = ad_mask(ctxt);
658 masked_increment(reg, mask, inc);
661 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
663 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
666 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
668 register_address_increment(ctxt, &ctxt->_eip, rel);
671 static u32 desc_limit_scaled(struct desc_struct *desc)
673 u32 limit = get_desc_limit(desc);
675 return desc->g ? (limit << 12) | 0xfff : limit;
678 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
680 ctxt->has_seg_override = true;
681 ctxt->seg_override = seg;
684 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
686 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
689 return ctxt->ops->get_cached_segment_base(ctxt, seg);
692 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
694 if (!ctxt->has_seg_override)
697 return ctxt->seg_override;
700 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
701 u32 error, bool valid)
703 ctxt->exception.vector = vec;
704 ctxt->exception.error_code = error;
705 ctxt->exception.error_code_valid = valid;
706 return X86EMUL_PROPAGATE_FAULT;
709 static int emulate_db(struct x86_emulate_ctxt *ctxt)
711 return emulate_exception(ctxt, DB_VECTOR, 0, false);
714 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
716 return emulate_exception(ctxt, GP_VECTOR, err, true);
719 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
721 return emulate_exception(ctxt, SS_VECTOR, err, true);
724 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
726 return emulate_exception(ctxt, UD_VECTOR, 0, false);
729 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
731 return emulate_exception(ctxt, TS_VECTOR, err, true);
734 static int emulate_de(struct x86_emulate_ctxt *ctxt)
736 return emulate_exception(ctxt, DE_VECTOR, 0, false);
739 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
741 return emulate_exception(ctxt, NM_VECTOR, 0, false);
744 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
747 struct desc_struct desc;
749 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
753 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
758 struct desc_struct desc;
760 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
761 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
765 * x86 defines three classes of vector instructions: explicitly
766 * aligned, explicitly unaligned, and the rest, which change behaviour
767 * depending on whether they're AVX encoded or not.
769 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
770 * subject to the same check.
772 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
774 if (likely(size < 16))
777 if (ctxt->d & Aligned)
779 else if (ctxt->d & Unaligned)
781 else if (ctxt->d & Avx)
787 static int __linearize(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 unsigned size, bool write, bool fetch,
792 struct desc_struct desc;
799 la = seg_base(ctxt, addr.seg) + addr.ea;
800 switch (ctxt->mode) {
801 case X86EMUL_MODE_PROT64:
802 if (((signed long)la << 16) >> 16 != la)
803 return emulate_gp(ctxt, 0);
806 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
810 /* code segment in protected mode or read-only data segment */
811 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
812 || !(desc.type & 2)) && write)
814 /* unreadable code segment */
815 if (!fetch && (desc.type & 8) && !(desc.type & 2))
817 lim = desc_limit_scaled(&desc);
818 if ((desc.type & 8) || !(desc.type & 4)) {
819 /* expand-up segment */
820 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
823 /* expand-down segment */
824 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
826 lim = desc.d ? 0xffffffff : 0xffff;
827 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
830 cpl = ctxt->ops->cpl(ctxt);
831 if (!(desc.type & 8)) {
835 } else if ((desc.type & 8) && !(desc.type & 4)) {
836 /* nonconforming code segment */
839 } else if ((desc.type & 8) && (desc.type & 4)) {
840 /* conforming code segment */
846 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
848 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
849 return emulate_gp(ctxt, 0);
851 return X86EMUL_CONTINUE;
853 if (addr.seg == VCPU_SREG_SS)
854 return emulate_ss(ctxt, sel);
856 return emulate_gp(ctxt, sel);
859 static int linearize(struct x86_emulate_ctxt *ctxt,
860 struct segmented_address addr,
861 unsigned size, bool write,
864 return __linearize(ctxt, addr, size, write, false, linear);
868 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
869 struct segmented_address addr,
876 rc = linearize(ctxt, addr, size, false, &linear);
877 if (rc != X86EMUL_CONTINUE)
879 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
883 * Fetch the next byte of the instruction being emulated which is pointed to
884 * by ctxt->_eip, then increment ctxt->_eip.
886 * Also prefetch the remaining bytes of the instruction without crossing page
887 * boundary if they are not in fetch_cache yet.
889 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
891 struct fetch_cache *fc = &ctxt->fetch;
895 if (ctxt->_eip == fc->end) {
896 unsigned long linear;
897 struct segmented_address addr = { .seg = VCPU_SREG_CS,
899 cur_size = fc->end - fc->start;
900 size = min(15UL - cur_size,
901 PAGE_SIZE - offset_in_page(ctxt->_eip));
902 rc = __linearize(ctxt, addr, size, false, true, &linear);
903 if (unlikely(rc != X86EMUL_CONTINUE))
905 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
906 size, &ctxt->exception);
907 if (unlikely(rc != X86EMUL_CONTINUE))
911 *dest = fc->data[ctxt->_eip - fc->start];
913 return X86EMUL_CONTINUE;
916 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
917 void *dest, unsigned size)
921 /* x86 instructions are limited to 15 bytes. */
922 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
923 return X86EMUL_UNHANDLEABLE;
925 rc = do_insn_fetch_byte(ctxt, dest++);
926 if (rc != X86EMUL_CONTINUE)
929 return X86EMUL_CONTINUE;
932 /* Fetch next part of the instruction being emulated. */
933 #define insn_fetch(_type, _ctxt) \
934 ({ unsigned long _x; \
935 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
936 if (rc != X86EMUL_CONTINUE) \
941 #define insn_fetch_arr(_arr, _size, _ctxt) \
942 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
943 if (rc != X86EMUL_CONTINUE) \
948 * Given the 'reg' portion of a ModRM byte, and a register block, return a
949 * pointer into the block that addresses the relevant register.
950 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
952 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
957 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
958 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
960 p = reg_rmw(ctxt, modrm_reg);
964 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
965 struct segmented_address addr,
966 u16 *size, unsigned long *address, int op_bytes)
973 rc = segmented_read_std(ctxt, addr, size, 2);
974 if (rc != X86EMUL_CONTINUE)
977 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1016 static u8 test_cc(unsigned int condition, unsigned long flags)
1019 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1021 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1022 asm("push %[flags]; popf; call *%[fastop]"
1023 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1027 static void fetch_register_operand(struct operand *op)
1029 switch (op->bytes) {
1031 op->val = *(u8 *)op->addr.reg;
1034 op->val = *(u16 *)op->addr.reg;
1037 op->val = *(u32 *)op->addr.reg;
1040 op->val = *(u64 *)op->addr.reg;
1045 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1047 ctxt->ops->get_fpu(ctxt);
1049 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1050 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1051 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1052 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1053 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1054 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1055 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1056 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1057 #ifdef CONFIG_X86_64
1058 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1059 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1060 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1061 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1062 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1063 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1064 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1065 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1069 ctxt->ops->put_fpu(ctxt);
1072 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1075 ctxt->ops->get_fpu(ctxt);
1077 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1078 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1079 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1080 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1081 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1082 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1083 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1084 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1085 #ifdef CONFIG_X86_64
1086 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1087 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1088 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1089 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1090 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1091 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1092 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1093 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1097 ctxt->ops->put_fpu(ctxt);
1100 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1102 ctxt->ops->get_fpu(ctxt);
1104 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1105 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1106 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1107 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1108 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1109 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1110 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1111 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1114 ctxt->ops->put_fpu(ctxt);
1117 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1119 ctxt->ops->get_fpu(ctxt);
1121 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1122 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1123 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1124 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1125 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1126 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1127 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1128 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1131 ctxt->ops->put_fpu(ctxt);
1134 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1136 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1137 return emulate_nm(ctxt);
1139 ctxt->ops->get_fpu(ctxt);
1140 asm volatile("fninit");
1141 ctxt->ops->put_fpu(ctxt);
1142 return X86EMUL_CONTINUE;
1145 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1149 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1150 return emulate_nm(ctxt);
1152 ctxt->ops->get_fpu(ctxt);
1153 asm volatile("fnstcw %0": "+m"(fcw));
1154 ctxt->ops->put_fpu(ctxt);
1156 /* force 2 byte destination */
1157 ctxt->dst.bytes = 2;
1158 ctxt->dst.val = fcw;
1160 return X86EMUL_CONTINUE;
1163 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1167 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1168 return emulate_nm(ctxt);
1170 ctxt->ops->get_fpu(ctxt);
1171 asm volatile("fnstsw %0": "+m"(fsw));
1172 ctxt->ops->put_fpu(ctxt);
1174 /* force 2 byte destination */
1175 ctxt->dst.bytes = 2;
1176 ctxt->dst.val = fsw;
1178 return X86EMUL_CONTINUE;
1181 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1184 unsigned reg = ctxt->modrm_reg;
1185 int highbyte_regs = ctxt->rex_prefix == 0;
1187 if (!(ctxt->d & ModRM))
1188 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1190 if (ctxt->d & Sse) {
1194 read_sse_reg(ctxt, &op->vec_val, reg);
1197 if (ctxt->d & Mmx) {
1206 if (ctxt->d & ByteOp) {
1207 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1210 op->addr.reg = decode_register(ctxt, reg, 0);
1211 op->bytes = ctxt->op_bytes;
1213 fetch_register_operand(op);
1214 op->orig_val = op->val;
1217 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1219 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1220 ctxt->modrm_seg = VCPU_SREG_SS;
1223 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1227 int index_reg = 0, base_reg = 0, scale;
1228 int rc = X86EMUL_CONTINUE;
1231 if (ctxt->rex_prefix) {
1232 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1233 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1234 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1237 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1238 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1239 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1240 ctxt->modrm_seg = VCPU_SREG_DS;
1242 if (ctxt->modrm_mod == 3) {
1243 int highbyte_regs = ctxt->rex_prefix == 0;
1246 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1247 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1248 highbyte_regs && (ctxt->d & ByteOp));
1249 if (ctxt->d & Sse) {
1252 op->addr.xmm = ctxt->modrm_rm;
1253 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1256 if (ctxt->d & Mmx) {
1259 op->addr.xmm = ctxt->modrm_rm & 7;
1262 fetch_register_operand(op);
1268 if (ctxt->ad_bytes == 2) {
1269 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1270 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1271 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1272 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1274 /* 16-bit ModR/M decode. */
1275 switch (ctxt->modrm_mod) {
1277 if (ctxt->modrm_rm == 6)
1278 modrm_ea += insn_fetch(u16, ctxt);
1281 modrm_ea += insn_fetch(s8, ctxt);
1284 modrm_ea += insn_fetch(u16, ctxt);
1287 switch (ctxt->modrm_rm) {
1289 modrm_ea += bx + si;
1292 modrm_ea += bx + di;
1295 modrm_ea += bp + si;
1298 modrm_ea += bp + di;
1307 if (ctxt->modrm_mod != 0)
1314 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1315 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1316 ctxt->modrm_seg = VCPU_SREG_SS;
1317 modrm_ea = (u16)modrm_ea;
1319 /* 32/64-bit ModR/M decode. */
1320 if ((ctxt->modrm_rm & 7) == 4) {
1321 sib = insn_fetch(u8, ctxt);
1322 index_reg |= (sib >> 3) & 7;
1323 base_reg |= sib & 7;
1326 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1327 modrm_ea += insn_fetch(s32, ctxt);
1329 modrm_ea += reg_read(ctxt, base_reg);
1330 adjust_modrm_seg(ctxt, base_reg);
1333 modrm_ea += reg_read(ctxt, index_reg) << scale;
1334 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1335 if (ctxt->mode == X86EMUL_MODE_PROT64)
1336 ctxt->rip_relative = 1;
1338 base_reg = ctxt->modrm_rm;
1339 modrm_ea += reg_read(ctxt, base_reg);
1340 adjust_modrm_seg(ctxt, base_reg);
1342 switch (ctxt->modrm_mod) {
1344 if (ctxt->modrm_rm == 5)
1345 modrm_ea += insn_fetch(s32, ctxt);
1348 modrm_ea += insn_fetch(s8, ctxt);
1351 modrm_ea += insn_fetch(s32, ctxt);
1355 op->addr.mem.ea = modrm_ea;
1360 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1363 int rc = X86EMUL_CONTINUE;
1366 switch (ctxt->ad_bytes) {
1368 op->addr.mem.ea = insn_fetch(u16, ctxt);
1371 op->addr.mem.ea = insn_fetch(u32, ctxt);
1374 op->addr.mem.ea = insn_fetch(u64, ctxt);
1381 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1385 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1386 mask = ~(ctxt->dst.bytes * 8 - 1);
1388 if (ctxt->src.bytes == 2)
1389 sv = (s16)ctxt->src.val & (s16)mask;
1390 else if (ctxt->src.bytes == 4)
1391 sv = (s32)ctxt->src.val & (s32)mask;
1393 ctxt->dst.addr.mem.ea += (sv >> 3);
1396 /* only subword offset */
1397 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1400 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1401 unsigned long addr, void *dest, unsigned size)
1404 struct read_cache *mc = &ctxt->mem_read;
1406 if (mc->pos < mc->end)
1409 WARN_ON((mc->end + size) >= sizeof(mc->data));
1411 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1413 if (rc != X86EMUL_CONTINUE)
1419 memcpy(dest, mc->data + mc->pos, size);
1421 return X86EMUL_CONTINUE;
1424 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1425 struct segmented_address addr,
1432 rc = linearize(ctxt, addr, size, false, &linear);
1433 if (rc != X86EMUL_CONTINUE)
1435 return read_emulated(ctxt, linear, data, size);
1438 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1439 struct segmented_address addr,
1446 rc = linearize(ctxt, addr, size, true, &linear);
1447 if (rc != X86EMUL_CONTINUE)
1449 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1453 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1454 struct segmented_address addr,
1455 const void *orig_data, const void *data,
1461 rc = linearize(ctxt, addr, size, true, &linear);
1462 if (rc != X86EMUL_CONTINUE)
1464 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1465 size, &ctxt->exception);
1468 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1469 unsigned int size, unsigned short port,
1472 struct read_cache *rc = &ctxt->io_read;
1474 if (rc->pos == rc->end) { /* refill pio read ahead */
1475 unsigned int in_page, n;
1476 unsigned int count = ctxt->rep_prefix ?
1477 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1478 in_page = (ctxt->eflags & EFLG_DF) ?
1479 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1480 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1481 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1485 rc->pos = rc->end = 0;
1486 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1491 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1492 ctxt->dst.data = rc->data + rc->pos;
1493 ctxt->dst.type = OP_MEM_STR;
1494 ctxt->dst.count = (rc->end - rc->pos) / size;
1497 memcpy(dest, rc->data + rc->pos, size);
1503 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1504 u16 index, struct desc_struct *desc)
1509 ctxt->ops->get_idt(ctxt, &dt);
1511 if (dt.size < index * 8 + 7)
1512 return emulate_gp(ctxt, index << 3 | 0x2);
1514 addr = dt.address + index * 8;
1515 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1519 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1520 u16 selector, struct desc_ptr *dt)
1522 const struct x86_emulate_ops *ops = ctxt->ops;
1524 if (selector & 1 << 2) {
1525 struct desc_struct desc;
1528 memset (dt, 0, sizeof *dt);
1529 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1532 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1533 dt->address = get_desc_base(&desc);
1535 ops->get_gdt(ctxt, dt);
1538 /* allowed just for 8 bytes segments */
1539 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1540 u16 selector, struct desc_struct *desc,
1544 u16 index = selector >> 3;
1547 get_descriptor_table_ptr(ctxt, selector, &dt);
1549 if (dt.size < index * 8 + 7)
1550 return emulate_gp(ctxt, selector & 0xfffc);
1552 *desc_addr_p = addr = dt.address + index * 8;
1553 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1557 /* allowed just for 8 bytes segments */
1558 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1559 u16 selector, struct desc_struct *desc)
1562 u16 index = selector >> 3;
1565 get_descriptor_table_ptr(ctxt, selector, &dt);
1567 if (dt.size < index * 8 + 7)
1568 return emulate_gp(ctxt, selector & 0xfffc);
1570 addr = dt.address + index * 8;
1571 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1575 /* Does not support long mode */
1576 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1577 u16 selector, int seg)
1579 struct desc_struct seg_desc, old_desc;
1581 unsigned err_vec = GP_VECTOR;
1583 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1588 memset(&seg_desc, 0, sizeof seg_desc);
1590 if (ctxt->mode == X86EMUL_MODE_REAL) {
1591 /* set real mode segment descriptor (keep limit etc. for
1593 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1594 set_desc_base(&seg_desc, selector << 4);
1596 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1597 /* VM86 needs a clean new segment descriptor */
1598 set_desc_base(&seg_desc, selector << 4);
1599 set_desc_limit(&seg_desc, 0xffff);
1608 cpl = ctxt->ops->cpl(ctxt);
1610 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1611 if ((seg == VCPU_SREG_CS
1612 || (seg == VCPU_SREG_SS
1613 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1614 || seg == VCPU_SREG_TR)
1618 /* TR should be in GDT only */
1619 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1622 if (null_selector) /* for NULL selector skip all following checks */
1625 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1626 if (ret != X86EMUL_CONTINUE)
1629 err_code = selector & 0xfffc;
1630 err_vec = GP_VECTOR;
1632 /* can't load system descriptor into segment selector */
1633 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1637 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1646 * segment is not a writable data segment or segment
1647 * selector's RPL != CPL or segment selector's RPL != CPL
1649 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1653 if (!(seg_desc.type & 8))
1656 if (seg_desc.type & 4) {
1662 if (rpl > cpl || dpl != cpl)
1665 /* CS(RPL) <- CPL */
1666 selector = (selector & 0xfffc) | cpl;
1669 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1671 old_desc = seg_desc;
1672 seg_desc.type |= 2; /* busy */
1673 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1674 sizeof(seg_desc), &ctxt->exception);
1675 if (ret != X86EMUL_CONTINUE)
1678 case VCPU_SREG_LDTR:
1679 if (seg_desc.s || seg_desc.type != 2)
1682 default: /* DS, ES, FS, or GS */
1684 * segment is not a data or readable code segment or
1685 * ((segment is a data or nonconforming code segment)
1686 * and (both RPL and CPL > DPL))
1688 if ((seg_desc.type & 0xa) == 0x8 ||
1689 (((seg_desc.type & 0xc) != 0xc) &&
1690 (rpl > dpl && cpl > dpl)))
1696 /* mark segment as accessed */
1698 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1699 if (ret != X86EMUL_CONTINUE)
1703 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1704 return X86EMUL_CONTINUE;
1706 emulate_exception(ctxt, err_vec, err_code, true);
1707 return X86EMUL_PROPAGATE_FAULT;
1710 static void write_register_operand(struct operand *op)
1712 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1713 switch (op->bytes) {
1715 *(u8 *)op->addr.reg = (u8)op->val;
1718 *(u16 *)op->addr.reg = (u16)op->val;
1721 *op->addr.reg = (u32)op->val;
1722 break; /* 64b: zero-extend */
1724 *op->addr.reg = op->val;
1729 static int writeback(struct x86_emulate_ctxt *ctxt)
1733 if (ctxt->d & NoWrite)
1734 return X86EMUL_CONTINUE;
1736 switch (ctxt->dst.type) {
1738 write_register_operand(&ctxt->dst);
1741 if (ctxt->lock_prefix)
1742 rc = segmented_cmpxchg(ctxt,
1744 &ctxt->dst.orig_val,
1748 rc = segmented_write(ctxt,
1752 if (rc != X86EMUL_CONTINUE)
1756 rc = segmented_write(ctxt,
1759 ctxt->dst.bytes * ctxt->dst.count);
1760 if (rc != X86EMUL_CONTINUE)
1764 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1767 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1775 return X86EMUL_CONTINUE;
1778 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1780 struct segmented_address addr;
1782 rsp_increment(ctxt, -bytes);
1783 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1784 addr.seg = VCPU_SREG_SS;
1786 return segmented_write(ctxt, addr, data, bytes);
1789 static int em_push(struct x86_emulate_ctxt *ctxt)
1791 /* Disable writeback. */
1792 ctxt->dst.type = OP_NONE;
1793 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1796 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1797 void *dest, int len)
1800 struct segmented_address addr;
1802 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1803 addr.seg = VCPU_SREG_SS;
1804 rc = segmented_read(ctxt, addr, dest, len);
1805 if (rc != X86EMUL_CONTINUE)
1808 rsp_increment(ctxt, len);
1812 static int em_pop(struct x86_emulate_ctxt *ctxt)
1814 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1817 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1818 void *dest, int len)
1821 unsigned long val, change_mask;
1822 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1823 int cpl = ctxt->ops->cpl(ctxt);
1825 rc = emulate_pop(ctxt, &val, len);
1826 if (rc != X86EMUL_CONTINUE)
1829 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1830 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1832 switch(ctxt->mode) {
1833 case X86EMUL_MODE_PROT64:
1834 case X86EMUL_MODE_PROT32:
1835 case X86EMUL_MODE_PROT16:
1837 change_mask |= EFLG_IOPL;
1839 change_mask |= EFLG_IF;
1841 case X86EMUL_MODE_VM86:
1843 return emulate_gp(ctxt, 0);
1844 change_mask |= EFLG_IF;
1846 default: /* real mode */
1847 change_mask |= (EFLG_IOPL | EFLG_IF);
1851 *(unsigned long *)dest =
1852 (ctxt->eflags & ~change_mask) | (val & change_mask);
1857 static int em_popf(struct x86_emulate_ctxt *ctxt)
1859 ctxt->dst.type = OP_REG;
1860 ctxt->dst.addr.reg = &ctxt->eflags;
1861 ctxt->dst.bytes = ctxt->op_bytes;
1862 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1865 static int em_enter(struct x86_emulate_ctxt *ctxt)
1868 unsigned frame_size = ctxt->src.val;
1869 unsigned nesting_level = ctxt->src2.val & 31;
1873 return X86EMUL_UNHANDLEABLE;
1875 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1876 rc = push(ctxt, &rbp, stack_size(ctxt));
1877 if (rc != X86EMUL_CONTINUE)
1879 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1881 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1882 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1884 return X86EMUL_CONTINUE;
1887 static int em_leave(struct x86_emulate_ctxt *ctxt)
1889 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1891 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1894 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1896 int seg = ctxt->src2.val;
1898 ctxt->src.val = get_segment_selector(ctxt, seg);
1900 return em_push(ctxt);
1903 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1905 int seg = ctxt->src2.val;
1906 unsigned long selector;
1909 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1910 if (rc != X86EMUL_CONTINUE)
1913 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1917 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1919 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1920 int rc = X86EMUL_CONTINUE;
1921 int reg = VCPU_REGS_RAX;
1923 while (reg <= VCPU_REGS_RDI) {
1924 (reg == VCPU_REGS_RSP) ?
1925 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1928 if (rc != X86EMUL_CONTINUE)
1937 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1939 ctxt->src.val = (unsigned long)ctxt->eflags;
1940 return em_push(ctxt);
1943 static int em_popa(struct x86_emulate_ctxt *ctxt)
1945 int rc = X86EMUL_CONTINUE;
1946 int reg = VCPU_REGS_RDI;
1948 while (reg >= VCPU_REGS_RAX) {
1949 if (reg == VCPU_REGS_RSP) {
1950 rsp_increment(ctxt, ctxt->op_bytes);
1954 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1955 if (rc != X86EMUL_CONTINUE)
1962 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1964 const struct x86_emulate_ops *ops = ctxt->ops;
1971 /* TODO: Add limit checks */
1972 ctxt->src.val = ctxt->eflags;
1974 if (rc != X86EMUL_CONTINUE)
1977 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1979 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1981 if (rc != X86EMUL_CONTINUE)
1984 ctxt->src.val = ctxt->_eip;
1986 if (rc != X86EMUL_CONTINUE)
1989 ops->get_idt(ctxt, &dt);
1991 eip_addr = dt.address + (irq << 2);
1992 cs_addr = dt.address + (irq << 2) + 2;
1994 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1995 if (rc != X86EMUL_CONTINUE)
1998 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1999 if (rc != X86EMUL_CONTINUE)
2002 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2003 if (rc != X86EMUL_CONTINUE)
2011 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2015 invalidate_registers(ctxt);
2016 rc = __emulate_int_real(ctxt, irq);
2017 if (rc == X86EMUL_CONTINUE)
2018 writeback_registers(ctxt);
2022 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2024 switch(ctxt->mode) {
2025 case X86EMUL_MODE_REAL:
2026 return __emulate_int_real(ctxt, irq);
2027 case X86EMUL_MODE_VM86:
2028 case X86EMUL_MODE_PROT16:
2029 case X86EMUL_MODE_PROT32:
2030 case X86EMUL_MODE_PROT64:
2032 /* Protected mode interrupts unimplemented yet */
2033 return X86EMUL_UNHANDLEABLE;
2037 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2039 int rc = X86EMUL_CONTINUE;
2040 unsigned long temp_eip = 0;
2041 unsigned long temp_eflags = 0;
2042 unsigned long cs = 0;
2043 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2044 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2045 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2046 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2048 /* TODO: Add stack limit check */
2050 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2052 if (rc != X86EMUL_CONTINUE)
2055 if (temp_eip & ~0xffff)
2056 return emulate_gp(ctxt, 0);
2058 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2060 if (rc != X86EMUL_CONTINUE)
2063 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2065 if (rc != X86EMUL_CONTINUE)
2068 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2070 if (rc != X86EMUL_CONTINUE)
2073 ctxt->_eip = temp_eip;
2076 if (ctxt->op_bytes == 4)
2077 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2078 else if (ctxt->op_bytes == 2) {
2079 ctxt->eflags &= ~0xffff;
2080 ctxt->eflags |= temp_eflags;
2083 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2084 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2089 static int em_iret(struct x86_emulate_ctxt *ctxt)
2091 switch(ctxt->mode) {
2092 case X86EMUL_MODE_REAL:
2093 return emulate_iret_real(ctxt);
2094 case X86EMUL_MODE_VM86:
2095 case X86EMUL_MODE_PROT16:
2096 case X86EMUL_MODE_PROT32:
2097 case X86EMUL_MODE_PROT64:
2099 /* iret from protected mode unimplemented yet */
2100 return X86EMUL_UNHANDLEABLE;
2104 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2109 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2111 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2112 if (rc != X86EMUL_CONTINUE)
2116 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2117 return X86EMUL_CONTINUE;
2120 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2124 emulate_1op_rax_rdx(ctxt, "mul", ex);
2125 return X86EMUL_CONTINUE;
2128 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2132 emulate_1op_rax_rdx(ctxt, "imul", ex);
2133 return X86EMUL_CONTINUE;
2136 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2140 emulate_1op_rax_rdx(ctxt, "div", de);
2142 return emulate_de(ctxt);
2143 return X86EMUL_CONTINUE;
2146 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2150 emulate_1op_rax_rdx(ctxt, "idiv", de);
2152 return emulate_de(ctxt);
2153 return X86EMUL_CONTINUE;
2156 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2158 int rc = X86EMUL_CONTINUE;
2160 switch (ctxt->modrm_reg) {
2161 case 2: /* call near abs */ {
2163 old_eip = ctxt->_eip;
2164 ctxt->_eip = ctxt->src.val;
2165 ctxt->src.val = old_eip;
2169 case 4: /* jmp abs */
2170 ctxt->_eip = ctxt->src.val;
2172 case 5: /* jmp far */
2173 rc = em_jmp_far(ctxt);
2182 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2184 u64 old = ctxt->dst.orig_val64;
2186 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2187 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2188 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2189 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2190 ctxt->eflags &= ~EFLG_ZF;
2192 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2193 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2195 ctxt->eflags |= EFLG_ZF;
2197 return X86EMUL_CONTINUE;
2200 static int em_ret(struct x86_emulate_ctxt *ctxt)
2202 ctxt->dst.type = OP_REG;
2203 ctxt->dst.addr.reg = &ctxt->_eip;
2204 ctxt->dst.bytes = ctxt->op_bytes;
2205 return em_pop(ctxt);
2208 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2213 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2214 if (rc != X86EMUL_CONTINUE)
2216 if (ctxt->op_bytes == 4)
2217 ctxt->_eip = (u32)ctxt->_eip;
2218 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2219 if (rc != X86EMUL_CONTINUE)
2221 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2225 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2227 /* Save real source value, then compare EAX against destination. */
2228 ctxt->src.orig_val = ctxt->src.val;
2229 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2230 fastop(ctxt, em_cmp);
2232 if (ctxt->eflags & EFLG_ZF) {
2233 /* Success: write back to memory. */
2234 ctxt->dst.val = ctxt->src.orig_val;
2236 /* Failure: write the value we saw to EAX. */
2237 ctxt->dst.type = OP_REG;
2238 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2240 return X86EMUL_CONTINUE;
2243 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2245 int seg = ctxt->src2.val;
2249 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2251 rc = load_segment_descriptor(ctxt, sel, seg);
2252 if (rc != X86EMUL_CONTINUE)
2255 ctxt->dst.val = ctxt->src.val;
2260 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2261 struct desc_struct *cs, struct desc_struct *ss)
2263 cs->l = 0; /* will be adjusted later */
2264 set_desc_base(cs, 0); /* flat segment */
2265 cs->g = 1; /* 4kb granularity */
2266 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2267 cs->type = 0x0b; /* Read, Execute, Accessed */
2269 cs->dpl = 0; /* will be adjusted later */
2274 set_desc_base(ss, 0); /* flat segment */
2275 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2276 ss->g = 1; /* 4kb granularity */
2278 ss->type = 0x03; /* Read/Write, Accessed */
2279 ss->d = 1; /* 32bit stack segment */
2286 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2288 u32 eax, ebx, ecx, edx;
2291 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2292 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2293 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2294 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2297 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2299 const struct x86_emulate_ops *ops = ctxt->ops;
2300 u32 eax, ebx, ecx, edx;
2303 * syscall should always be enabled in longmode - so only become
2304 * vendor specific (cpuid) if other modes are active...
2306 if (ctxt->mode == X86EMUL_MODE_PROT64)
2311 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2313 * Intel ("GenuineIntel")
2314 * remark: Intel CPUs only support "syscall" in 64bit
2315 * longmode. Also an 64bit guest with a
2316 * 32bit compat-app running will #UD !! While this
2317 * behaviour can be fixed (by emulating) into AMD
2318 * response - CPUs of AMD can't behave like Intel.
2320 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2321 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2322 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2325 /* AMD ("AuthenticAMD") */
2326 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2327 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2328 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2331 /* AMD ("AMDisbetter!") */
2332 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2333 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2334 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2337 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2341 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2343 const struct x86_emulate_ops *ops = ctxt->ops;
2344 struct desc_struct cs, ss;
2349 /* syscall is not available in real mode */
2350 if (ctxt->mode == X86EMUL_MODE_REAL ||
2351 ctxt->mode == X86EMUL_MODE_VM86)
2352 return emulate_ud(ctxt);
2354 if (!(em_syscall_is_enabled(ctxt)))
2355 return emulate_ud(ctxt);
2357 ops->get_msr(ctxt, MSR_EFER, &efer);
2358 setup_syscalls_segments(ctxt, &cs, &ss);
2360 if (!(efer & EFER_SCE))
2361 return emulate_ud(ctxt);
2363 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2365 cs_sel = (u16)(msr_data & 0xfffc);
2366 ss_sel = (u16)(msr_data + 8);
2368 if (efer & EFER_LMA) {
2372 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2373 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2375 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2376 if (efer & EFER_LMA) {
2377 #ifdef CONFIG_X86_64
2378 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2381 ctxt->mode == X86EMUL_MODE_PROT64 ?
2382 MSR_LSTAR : MSR_CSTAR, &msr_data);
2383 ctxt->_eip = msr_data;
2385 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2386 ctxt->eflags &= ~(msr_data | EFLG_RF);
2390 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2391 ctxt->_eip = (u32)msr_data;
2393 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2396 return X86EMUL_CONTINUE;
2399 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2401 const struct x86_emulate_ops *ops = ctxt->ops;
2402 struct desc_struct cs, ss;
2407 ops->get_msr(ctxt, MSR_EFER, &efer);
2408 /* inject #GP if in real mode */
2409 if (ctxt->mode == X86EMUL_MODE_REAL)
2410 return emulate_gp(ctxt, 0);
2413 * Not recognized on AMD in compat mode (but is recognized in legacy
2416 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2417 && !vendor_intel(ctxt))
2418 return emulate_ud(ctxt);
2420 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2421 * Therefore, we inject an #UD.
2423 if (ctxt->mode == X86EMUL_MODE_PROT64)
2424 return emulate_ud(ctxt);
2426 setup_syscalls_segments(ctxt, &cs, &ss);
2428 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2429 switch (ctxt->mode) {
2430 case X86EMUL_MODE_PROT32:
2431 if ((msr_data & 0xfffc) == 0x0)
2432 return emulate_gp(ctxt, 0);
2434 case X86EMUL_MODE_PROT64:
2435 if (msr_data == 0x0)
2436 return emulate_gp(ctxt, 0);
2442 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2443 cs_sel = (u16)msr_data;
2444 cs_sel &= ~SELECTOR_RPL_MASK;
2445 ss_sel = cs_sel + 8;
2446 ss_sel &= ~SELECTOR_RPL_MASK;
2447 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2452 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2453 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2455 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2456 ctxt->_eip = msr_data;
2458 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2459 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2461 return X86EMUL_CONTINUE;
2464 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2466 const struct x86_emulate_ops *ops = ctxt->ops;
2467 struct desc_struct cs, ss;
2470 u16 cs_sel = 0, ss_sel = 0;
2472 /* inject #GP if in real mode or Virtual 8086 mode */
2473 if (ctxt->mode == X86EMUL_MODE_REAL ||
2474 ctxt->mode == X86EMUL_MODE_VM86)
2475 return emulate_gp(ctxt, 0);
2477 setup_syscalls_segments(ctxt, &cs, &ss);
2479 if ((ctxt->rex_prefix & 0x8) != 0x0)
2480 usermode = X86EMUL_MODE_PROT64;
2482 usermode = X86EMUL_MODE_PROT32;
2486 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2488 case X86EMUL_MODE_PROT32:
2489 cs_sel = (u16)(msr_data + 16);
2490 if ((msr_data & 0xfffc) == 0x0)
2491 return emulate_gp(ctxt, 0);
2492 ss_sel = (u16)(msr_data + 24);
2494 case X86EMUL_MODE_PROT64:
2495 cs_sel = (u16)(msr_data + 32);
2496 if (msr_data == 0x0)
2497 return emulate_gp(ctxt, 0);
2498 ss_sel = cs_sel + 8;
2503 cs_sel |= SELECTOR_RPL_MASK;
2504 ss_sel |= SELECTOR_RPL_MASK;
2506 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2507 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2509 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2510 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2512 return X86EMUL_CONTINUE;
2515 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2518 if (ctxt->mode == X86EMUL_MODE_REAL)
2520 if (ctxt->mode == X86EMUL_MODE_VM86)
2522 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2523 return ctxt->ops->cpl(ctxt) > iopl;
2526 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2529 const struct x86_emulate_ops *ops = ctxt->ops;
2530 struct desc_struct tr_seg;
2533 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2534 unsigned mask = (1 << len) - 1;
2537 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2540 if (desc_limit_scaled(&tr_seg) < 103)
2542 base = get_desc_base(&tr_seg);
2543 #ifdef CONFIG_X86_64
2544 base |= ((u64)base3) << 32;
2546 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2547 if (r != X86EMUL_CONTINUE)
2549 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2551 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2552 if (r != X86EMUL_CONTINUE)
2554 if ((perm >> bit_idx) & mask)
2559 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2565 if (emulator_bad_iopl(ctxt))
2566 if (!emulator_io_port_access_allowed(ctxt, port, len))
2569 ctxt->perm_ok = true;
2574 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2575 struct tss_segment_16 *tss)
2577 tss->ip = ctxt->_eip;
2578 tss->flag = ctxt->eflags;
2579 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2580 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2581 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2582 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2583 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2584 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2585 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2586 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2588 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2589 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2590 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2591 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2592 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2595 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2596 struct tss_segment_16 *tss)
2600 ctxt->_eip = tss->ip;
2601 ctxt->eflags = tss->flag | 2;
2602 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2603 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2604 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2605 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2606 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2607 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2608 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2609 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2612 * SDM says that segment selectors are loaded before segment
2615 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2616 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2617 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2618 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2619 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2622 * Now load segment descriptors. If fault happens at this stage
2623 * it is handled in a context of new task
2625 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2626 if (ret != X86EMUL_CONTINUE)
2628 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2635 if (ret != X86EMUL_CONTINUE)
2637 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2638 if (ret != X86EMUL_CONTINUE)
2641 return X86EMUL_CONTINUE;
2644 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2645 u16 tss_selector, u16 old_tss_sel,
2646 ulong old_tss_base, struct desc_struct *new_desc)
2648 const struct x86_emulate_ops *ops = ctxt->ops;
2649 struct tss_segment_16 tss_seg;
2651 u32 new_tss_base = get_desc_base(new_desc);
2653 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2655 if (ret != X86EMUL_CONTINUE)
2656 /* FIXME: need to provide precise fault address */
2659 save_state_to_tss16(ctxt, &tss_seg);
2661 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2663 if (ret != X86EMUL_CONTINUE)
2664 /* FIXME: need to provide precise fault address */
2667 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2669 if (ret != X86EMUL_CONTINUE)
2670 /* FIXME: need to provide precise fault address */
2673 if (old_tss_sel != 0xffff) {
2674 tss_seg.prev_task_link = old_tss_sel;
2676 ret = ops->write_std(ctxt, new_tss_base,
2677 &tss_seg.prev_task_link,
2678 sizeof tss_seg.prev_task_link,
2680 if (ret != X86EMUL_CONTINUE)
2681 /* FIXME: need to provide precise fault address */
2685 return load_state_from_tss16(ctxt, &tss_seg);
2688 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2689 struct tss_segment_32 *tss)
2691 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2692 tss->eip = ctxt->_eip;
2693 tss->eflags = ctxt->eflags;
2694 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2695 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2696 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2697 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2698 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2699 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2700 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2701 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2703 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2704 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2705 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2706 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2707 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2708 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2709 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2712 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2713 struct tss_segment_32 *tss)
2717 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2718 return emulate_gp(ctxt, 0);
2719 ctxt->_eip = tss->eip;
2720 ctxt->eflags = tss->eflags | 2;
2722 /* General purpose registers */
2723 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2724 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2725 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2726 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2727 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2728 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2729 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2730 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2733 * SDM says that segment selectors are loaded before segment
2736 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2737 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2738 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2739 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2740 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2741 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2742 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2745 * If we're switching between Protected Mode and VM86, we need to make
2746 * sure to update the mode before loading the segment descriptors so
2747 * that the selectors are interpreted correctly.
2749 * Need to get rflags to the vcpu struct immediately because it
2750 * influences the CPL which is checked at least when loading the segment
2751 * descriptors and when pushing an error code to the new kernel stack.
2753 * TODO Introduce a separate ctxt->ops->set_cpl callback
2755 if (ctxt->eflags & X86_EFLAGS_VM)
2756 ctxt->mode = X86EMUL_MODE_VM86;
2758 ctxt->mode = X86EMUL_MODE_PROT32;
2760 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2763 * Now load segment descriptors. If fault happenes at this stage
2764 * it is handled in a context of new task
2766 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2767 if (ret != X86EMUL_CONTINUE)
2769 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2770 if (ret != X86EMUL_CONTINUE)
2772 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2776 if (ret != X86EMUL_CONTINUE)
2778 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2779 if (ret != X86EMUL_CONTINUE)
2781 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2782 if (ret != X86EMUL_CONTINUE)
2784 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2785 if (ret != X86EMUL_CONTINUE)
2788 return X86EMUL_CONTINUE;
2791 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2792 u16 tss_selector, u16 old_tss_sel,
2793 ulong old_tss_base, struct desc_struct *new_desc)
2795 const struct x86_emulate_ops *ops = ctxt->ops;
2796 struct tss_segment_32 tss_seg;
2798 u32 new_tss_base = get_desc_base(new_desc);
2800 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2802 if (ret != X86EMUL_CONTINUE)
2803 /* FIXME: need to provide precise fault address */
2806 save_state_to_tss32(ctxt, &tss_seg);
2808 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2810 if (ret != X86EMUL_CONTINUE)
2811 /* FIXME: need to provide precise fault address */
2814 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2816 if (ret != X86EMUL_CONTINUE)
2817 /* FIXME: need to provide precise fault address */
2820 if (old_tss_sel != 0xffff) {
2821 tss_seg.prev_task_link = old_tss_sel;
2823 ret = ops->write_std(ctxt, new_tss_base,
2824 &tss_seg.prev_task_link,
2825 sizeof tss_seg.prev_task_link,
2827 if (ret != X86EMUL_CONTINUE)
2828 /* FIXME: need to provide precise fault address */
2832 return load_state_from_tss32(ctxt, &tss_seg);
2835 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2836 u16 tss_selector, int idt_index, int reason,
2837 bool has_error_code, u32 error_code)
2839 const struct x86_emulate_ops *ops = ctxt->ops;
2840 struct desc_struct curr_tss_desc, next_tss_desc;
2842 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2843 ulong old_tss_base =
2844 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2848 /* FIXME: old_tss_base == ~0 ? */
2850 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2851 if (ret != X86EMUL_CONTINUE)
2853 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2854 if (ret != X86EMUL_CONTINUE)
2857 /* FIXME: check that next_tss_desc is tss */
2860 * Check privileges. The three cases are task switch caused by...
2862 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2863 * 2. Exception/IRQ/iret: No check is performed
2864 * 3. jmp/call to TSS: Check against DPL of the TSS
2866 if (reason == TASK_SWITCH_GATE) {
2867 if (idt_index != -1) {
2868 /* Software interrupts */
2869 struct desc_struct task_gate_desc;
2872 ret = read_interrupt_descriptor(ctxt, idt_index,
2874 if (ret != X86EMUL_CONTINUE)
2877 dpl = task_gate_desc.dpl;
2878 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2879 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2881 } else if (reason != TASK_SWITCH_IRET) {
2882 int dpl = next_tss_desc.dpl;
2883 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2884 return emulate_gp(ctxt, tss_selector);
2888 desc_limit = desc_limit_scaled(&next_tss_desc);
2889 if (!next_tss_desc.p ||
2890 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2891 desc_limit < 0x2b)) {
2892 emulate_ts(ctxt, tss_selector & 0xfffc);
2893 return X86EMUL_PROPAGATE_FAULT;
2896 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2897 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2898 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2901 if (reason == TASK_SWITCH_IRET)
2902 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2904 /* set back link to prev task only if NT bit is set in eflags
2905 note that old_tss_sel is not used after this point */
2906 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2907 old_tss_sel = 0xffff;
2909 if (next_tss_desc.type & 8)
2910 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2911 old_tss_base, &next_tss_desc);
2913 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2914 old_tss_base, &next_tss_desc);
2915 if (ret != X86EMUL_CONTINUE)
2918 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2919 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2921 if (reason != TASK_SWITCH_IRET) {
2922 next_tss_desc.type |= (1 << 1); /* set busy flag */
2923 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2926 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2927 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2929 if (has_error_code) {
2930 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2931 ctxt->lock_prefix = 0;
2932 ctxt->src.val = (unsigned long) error_code;
2933 ret = em_push(ctxt);
2939 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2940 u16 tss_selector, int idt_index, int reason,
2941 bool has_error_code, u32 error_code)
2945 invalidate_registers(ctxt);
2946 ctxt->_eip = ctxt->eip;
2947 ctxt->dst.type = OP_NONE;
2949 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2950 has_error_code, error_code);
2952 if (rc == X86EMUL_CONTINUE) {
2953 ctxt->eip = ctxt->_eip;
2954 writeback_registers(ctxt);
2957 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2960 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2963 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2965 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2966 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2969 static int em_das(struct x86_emulate_ctxt *ctxt)
2972 bool af, cf, old_cf;
2974 cf = ctxt->eflags & X86_EFLAGS_CF;
2980 af = ctxt->eflags & X86_EFLAGS_AF;
2981 if ((al & 0x0f) > 9 || af) {
2983 cf = old_cf | (al >= 250);
2988 if (old_al > 0x99 || old_cf) {
2994 /* Set PF, ZF, SF */
2995 ctxt->src.type = OP_IMM;
2997 ctxt->src.bytes = 1;
2998 fastop(ctxt, em_or);
2999 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3001 ctxt->eflags |= X86_EFLAGS_CF;
3003 ctxt->eflags |= X86_EFLAGS_AF;
3004 return X86EMUL_CONTINUE;
3007 static int em_aam(struct x86_emulate_ctxt *ctxt)
3011 if (ctxt->src.val == 0)
3012 return emulate_de(ctxt);
3014 al = ctxt->dst.val & 0xff;
3015 ah = al / ctxt->src.val;
3016 al %= ctxt->src.val;
3018 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3020 /* Set PF, ZF, SF */
3021 ctxt->src.type = OP_IMM;
3023 ctxt->src.bytes = 1;
3024 fastop(ctxt, em_or);
3026 return X86EMUL_CONTINUE;
3029 static int em_aad(struct x86_emulate_ctxt *ctxt)
3031 u8 al = ctxt->dst.val & 0xff;
3032 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3034 al = (al + (ah * ctxt->src.val)) & 0xff;
3036 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3038 /* Set PF, ZF, SF */
3039 ctxt->src.type = OP_IMM;
3041 ctxt->src.bytes = 1;
3042 fastop(ctxt, em_or);
3044 return X86EMUL_CONTINUE;
3047 static int em_call(struct x86_emulate_ctxt *ctxt)
3049 long rel = ctxt->src.val;
3051 ctxt->src.val = (unsigned long)ctxt->_eip;
3053 return em_push(ctxt);
3056 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3062 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3063 old_eip = ctxt->_eip;
3065 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3066 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3067 return X86EMUL_CONTINUE;
3070 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3072 ctxt->src.val = old_cs;
3074 if (rc != X86EMUL_CONTINUE)
3077 ctxt->src.val = old_eip;
3078 return em_push(ctxt);
3081 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3085 ctxt->dst.type = OP_REG;
3086 ctxt->dst.addr.reg = &ctxt->_eip;
3087 ctxt->dst.bytes = ctxt->op_bytes;
3088 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3089 if (rc != X86EMUL_CONTINUE)
3091 rsp_increment(ctxt, ctxt->src.val);
3092 return X86EMUL_CONTINUE;
3095 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3097 /* Write back the register source. */
3098 ctxt->src.val = ctxt->dst.val;
3099 write_register_operand(&ctxt->src);
3101 /* Write back the memory destination with implicit LOCK prefix. */
3102 ctxt->dst.val = ctxt->src.orig_val;
3103 ctxt->lock_prefix = 1;
3104 return X86EMUL_CONTINUE;
3107 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3109 ctxt->dst.val = ctxt->src2.val;
3110 return fastop(ctxt, em_imul);
3113 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3115 ctxt->dst.type = OP_REG;
3116 ctxt->dst.bytes = ctxt->src.bytes;
3117 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3118 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3120 return X86EMUL_CONTINUE;
3123 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3127 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3128 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3129 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3130 return X86EMUL_CONTINUE;
3133 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3137 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3138 return emulate_gp(ctxt, 0);
3139 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3140 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3141 return X86EMUL_CONTINUE;
3144 static int em_mov(struct x86_emulate_ctxt *ctxt)
3146 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3147 return X86EMUL_CONTINUE;
3150 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3152 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3153 return emulate_gp(ctxt, 0);
3155 /* Disable writeback. */
3156 ctxt->dst.type = OP_NONE;
3157 return X86EMUL_CONTINUE;
3160 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3164 if (ctxt->mode == X86EMUL_MODE_PROT64)
3165 val = ctxt->src.val & ~0ULL;
3167 val = ctxt->src.val & ~0U;
3169 /* #UD condition is already handled. */
3170 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3171 return emulate_gp(ctxt, 0);
3173 /* Disable writeback. */
3174 ctxt->dst.type = OP_NONE;
3175 return X86EMUL_CONTINUE;
3178 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3182 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3183 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3184 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3185 return emulate_gp(ctxt, 0);
3187 return X86EMUL_CONTINUE;
3190 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3194 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3195 return emulate_gp(ctxt, 0);
3197 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3198 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3199 return X86EMUL_CONTINUE;
3202 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3204 if (ctxt->modrm_reg > VCPU_SREG_GS)
3205 return emulate_ud(ctxt);
3207 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3208 return X86EMUL_CONTINUE;
3211 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3213 u16 sel = ctxt->src.val;
3215 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3216 return emulate_ud(ctxt);
3218 if (ctxt->modrm_reg == VCPU_SREG_SS)
3219 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3221 /* Disable writeback. */
3222 ctxt->dst.type = OP_NONE;
3223 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3226 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3228 u16 sel = ctxt->src.val;
3230 /* Disable writeback. */
3231 ctxt->dst.type = OP_NONE;
3232 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3235 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3237 u16 sel = ctxt->src.val;
3239 /* Disable writeback. */
3240 ctxt->dst.type = OP_NONE;
3241 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3244 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3249 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3250 if (rc == X86EMUL_CONTINUE)
3251 ctxt->ops->invlpg(ctxt, linear);
3252 /* Disable writeback. */
3253 ctxt->dst.type = OP_NONE;
3254 return X86EMUL_CONTINUE;
3257 static int em_clts(struct x86_emulate_ctxt *ctxt)
3261 cr0 = ctxt->ops->get_cr(ctxt, 0);
3263 ctxt->ops->set_cr(ctxt, 0, cr0);
3264 return X86EMUL_CONTINUE;
3267 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3271 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3272 return X86EMUL_UNHANDLEABLE;
3274 rc = ctxt->ops->fix_hypercall(ctxt);
3275 if (rc != X86EMUL_CONTINUE)
3278 /* Let the processor re-execute the fixed hypercall */
3279 ctxt->_eip = ctxt->eip;
3280 /* Disable writeback. */
3281 ctxt->dst.type = OP_NONE;
3282 return X86EMUL_CONTINUE;
3285 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3286 void (*get)(struct x86_emulate_ctxt *ctxt,
3287 struct desc_ptr *ptr))
3289 struct desc_ptr desc_ptr;
3291 if (ctxt->mode == X86EMUL_MODE_PROT64)
3293 get(ctxt, &desc_ptr);
3294 if (ctxt->op_bytes == 2) {
3296 desc_ptr.address &= 0x00ffffff;
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3300 return segmented_write(ctxt, ctxt->dst.addr.mem,
3301 &desc_ptr, 2 + ctxt->op_bytes);
3304 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3306 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3309 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3311 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3314 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3316 struct desc_ptr desc_ptr;
3319 if (ctxt->mode == X86EMUL_MODE_PROT64)
3321 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3322 &desc_ptr.size, &desc_ptr.address,
3324 if (rc != X86EMUL_CONTINUE)
3326 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3327 /* Disable writeback. */
3328 ctxt->dst.type = OP_NONE;
3329 return X86EMUL_CONTINUE;
3332 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3336 rc = ctxt->ops->fix_hypercall(ctxt);
3338 /* Disable writeback. */
3339 ctxt->dst.type = OP_NONE;
3343 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3345 struct desc_ptr desc_ptr;
3348 if (ctxt->mode == X86EMUL_MODE_PROT64)
3350 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3351 &desc_ptr.size, &desc_ptr.address,
3353 if (rc != X86EMUL_CONTINUE)
3355 ctxt->ops->set_idt(ctxt, &desc_ptr);
3356 /* Disable writeback. */
3357 ctxt->dst.type = OP_NONE;
3358 return X86EMUL_CONTINUE;
3361 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3363 ctxt->dst.bytes = 2;
3364 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3365 return X86EMUL_CONTINUE;
3368 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3370 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3371 | (ctxt->src.val & 0x0f));
3372 ctxt->dst.type = OP_NONE;
3373 return X86EMUL_CONTINUE;
3376 static int em_loop(struct x86_emulate_ctxt *ctxt)
3378 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3379 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3380 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3381 jmp_rel(ctxt, ctxt->src.val);
3383 return X86EMUL_CONTINUE;
3386 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3388 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3389 jmp_rel(ctxt, ctxt->src.val);
3391 return X86EMUL_CONTINUE;
3394 static int em_in(struct x86_emulate_ctxt *ctxt)
3396 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3398 return X86EMUL_IO_NEEDED;
3400 return X86EMUL_CONTINUE;
3403 static int em_out(struct x86_emulate_ctxt *ctxt)
3405 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3407 /* Disable writeback. */
3408 ctxt->dst.type = OP_NONE;
3409 return X86EMUL_CONTINUE;
3412 static int em_cli(struct x86_emulate_ctxt *ctxt)
3414 if (emulator_bad_iopl(ctxt))
3415 return emulate_gp(ctxt, 0);
3417 ctxt->eflags &= ~X86_EFLAGS_IF;
3418 return X86EMUL_CONTINUE;
3421 static int em_sti(struct x86_emulate_ctxt *ctxt)
3423 if (emulator_bad_iopl(ctxt))
3424 return emulate_gp(ctxt, 0);
3426 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3427 ctxt->eflags |= X86_EFLAGS_IF;
3428 return X86EMUL_CONTINUE;
3431 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3433 u32 eax, ebx, ecx, edx;
3435 eax = reg_read(ctxt, VCPU_REGS_RAX);
3436 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3437 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3438 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3439 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3440 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3441 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3442 return X86EMUL_CONTINUE;
3445 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3447 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3448 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3449 return X86EMUL_CONTINUE;
3452 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3454 switch (ctxt->op_bytes) {
3455 #ifdef CONFIG_X86_64
3457 asm("bswap %0" : "+r"(ctxt->dst.val));
3461 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3464 return X86EMUL_CONTINUE;
3467 static bool valid_cr(int nr)
3479 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3481 if (!valid_cr(ctxt->modrm_reg))
3482 return emulate_ud(ctxt);
3484 return X86EMUL_CONTINUE;
3487 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3489 u64 new_val = ctxt->src.val64;
3490 int cr = ctxt->modrm_reg;
3493 static u64 cr_reserved_bits[] = {
3494 0xffffffff00000000ULL,
3495 0, 0, 0, /* CR3 checked later */
3502 return emulate_ud(ctxt);
3504 if (new_val & cr_reserved_bits[cr])
3505 return emulate_gp(ctxt, 0);
3510 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3511 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3512 return emulate_gp(ctxt, 0);
3514 cr4 = ctxt->ops->get_cr(ctxt, 4);
3515 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3517 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3518 !(cr4 & X86_CR4_PAE))
3519 return emulate_gp(ctxt, 0);
3526 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3527 if (efer & EFER_LMA)
3528 rsvd = CR3_L_MODE_RESERVED_BITS;
3529 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3530 rsvd = CR3_PAE_RESERVED_BITS;
3531 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3532 rsvd = CR3_NONPAE_RESERVED_BITS;
3535 return emulate_gp(ctxt, 0);
3540 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3542 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3543 return emulate_gp(ctxt, 0);
3549 return X86EMUL_CONTINUE;
3552 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3556 ctxt->ops->get_dr(ctxt, 7, &dr7);
3558 /* Check if DR7.Global_Enable is set */
3559 return dr7 & (1 << 13);
3562 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3564 int dr = ctxt->modrm_reg;
3568 return emulate_ud(ctxt);
3570 cr4 = ctxt->ops->get_cr(ctxt, 4);
3571 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3572 return emulate_ud(ctxt);
3574 if (check_dr7_gd(ctxt))
3575 return emulate_db(ctxt);
3577 return X86EMUL_CONTINUE;
3580 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3582 u64 new_val = ctxt->src.val64;
3583 int dr = ctxt->modrm_reg;
3585 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3586 return emulate_gp(ctxt, 0);
3588 return check_dr_read(ctxt);
3591 static int check_svme(struct x86_emulate_ctxt *ctxt)
3595 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3597 if (!(efer & EFER_SVME))
3598 return emulate_ud(ctxt);
3600 return X86EMUL_CONTINUE;
3603 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3605 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3607 /* Valid physical address? */
3608 if (rax & 0xffff000000000000ULL)
3609 return emulate_gp(ctxt, 0);
3611 return check_svme(ctxt);
3614 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3616 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3618 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3619 return emulate_ud(ctxt);
3621 return X86EMUL_CONTINUE;
3624 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3626 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3627 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3629 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3631 return emulate_gp(ctxt, 0);
3633 return X86EMUL_CONTINUE;
3636 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3638 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3639 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3640 return emulate_gp(ctxt, 0);
3642 return X86EMUL_CONTINUE;
3645 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3647 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3648 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3649 return emulate_gp(ctxt, 0);
3651 return X86EMUL_CONTINUE;
3654 #define D(_y) { .flags = (_y) }
3655 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3656 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3657 .check_perm = (_p) }
3658 #define N D(NotImpl)
3659 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3660 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3661 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3662 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3663 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3664 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3665 #define II(_f, _e, _i) \
3666 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3667 #define IIP(_f, _e, _i, _p) \
3668 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3669 .check_perm = (_p) }
3670 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3672 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3673 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3674 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3675 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3676 #define I2bvIP(_f, _e, _i, _p) \
3677 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3679 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3680 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3681 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3683 static const struct opcode group7_rm1[] = {
3684 DI(SrcNone | Priv, monitor),
3685 DI(SrcNone | Priv, mwait),
3689 static const struct opcode group7_rm3[] = {
3690 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3691 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3692 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3693 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3694 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3695 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3696 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3697 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3700 static const struct opcode group7_rm7[] = {
3702 DIP(SrcNone, rdtscp, check_rdtsc),
3706 static const struct opcode group1[] = {
3708 F(Lock | PageTable, em_or),
3711 F(Lock | PageTable, em_and),
3717 static const struct opcode group1A[] = {
3718 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3721 static const struct opcode group2[] = {
3722 F(DstMem | ModRM, em_rol),
3723 F(DstMem | ModRM, em_ror),
3724 F(DstMem | ModRM, em_rcl),
3725 F(DstMem | ModRM, em_rcr),
3726 F(DstMem | ModRM, em_shl),
3727 F(DstMem | ModRM, em_shr),
3728 F(DstMem | ModRM, em_shl),
3729 F(DstMem | ModRM, em_sar),
3732 static const struct opcode group3[] = {
3733 F(DstMem | SrcImm | NoWrite, em_test),
3734 F(DstMem | SrcImm | NoWrite, em_test),
3735 F(DstMem | SrcNone | Lock, em_not),
3736 F(DstMem | SrcNone | Lock, em_neg),
3737 I(SrcMem, em_mul_ex),
3738 I(SrcMem, em_imul_ex),
3739 I(SrcMem, em_div_ex),
3740 I(SrcMem, em_idiv_ex),
3743 static const struct opcode group4[] = {
3744 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3745 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3749 static const struct opcode group5[] = {
3750 F(DstMem | SrcNone | Lock, em_inc),
3751 F(DstMem | SrcNone | Lock, em_dec),
3752 I(SrcMem | Stack, em_grp45),
3753 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3754 I(SrcMem | Stack, em_grp45),
3755 I(SrcMemFAddr | ImplicitOps, em_grp45),
3756 I(SrcMem | Stack, em_grp45), D(Undefined),
3759 static const struct opcode group6[] = {
3762 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3763 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3767 static const struct group_dual group7 = { {
3768 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3769 II(Mov | DstMem | Priv, em_sidt, sidt),
3770 II(SrcMem | Priv, em_lgdt, lgdt),
3771 II(SrcMem | Priv, em_lidt, lidt),
3772 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3773 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3774 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3776 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3778 N, EXT(0, group7_rm3),
3779 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3780 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3784 static const struct opcode group8[] = {
3786 F(DstMem | SrcImmByte | NoWrite, em_bt),
3787 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3788 F(DstMem | SrcImmByte | Lock, em_btr),
3789 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3792 static const struct group_dual group9 = { {
3793 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3795 N, N, N, N, N, N, N, N,
3798 static const struct opcode group11[] = {
3799 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3803 static const struct gprefix pfx_0f_6f_0f_7f = {
3804 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3807 static const struct gprefix pfx_vmovntpx = {
3808 I(0, em_mov), N, N, N,
3811 static const struct escape escape_d9 = { {
3812 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3815 N, N, N, N, N, N, N, N,
3817 N, N, N, N, N, N, N, N,
3819 N, N, N, N, N, N, N, N,
3821 N, N, N, N, N, N, N, N,
3823 N, N, N, N, N, N, N, N,
3825 N, N, N, N, N, N, N, N,
3827 N, N, N, N, N, N, N, N,
3829 N, N, N, N, N, N, N, N,
3832 static const struct escape escape_db = { {
3833 N, N, N, N, N, N, N, N,
3836 N, N, N, N, N, N, N, N,
3838 N, N, N, N, N, N, N, N,
3840 N, N, N, N, N, N, N, N,
3842 N, N, N, N, N, N, N, N,
3844 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3846 N, N, N, N, N, N, N, N,
3848 N, N, N, N, N, N, N, N,
3850 N, N, N, N, N, N, N, N,
3853 static const struct escape escape_dd = { {
3854 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3857 N, N, N, N, N, N, N, N,
3859 N, N, N, N, N, N, N, N,
3861 N, N, N, N, N, N, N, N,
3863 N, N, N, N, N, N, N, N,
3865 N, N, N, N, N, N, N, N,
3867 N, N, N, N, N, N, N, N,
3869 N, N, N, N, N, N, N, N,
3871 N, N, N, N, N, N, N, N,
3874 static const struct opcode opcode_table[256] = {
3876 F6ALU(Lock, em_add),
3877 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3878 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3880 F6ALU(Lock | PageTable, em_or),
3881 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3884 F6ALU(Lock, em_adc),
3885 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3886 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3888 F6ALU(Lock, em_sbb),
3889 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3890 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3892 F6ALU(Lock | PageTable, em_and), N, N,
3894 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3896 F6ALU(Lock, em_xor), N, N,
3898 F6ALU(NoWrite, em_cmp), N, N,
3900 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3902 X8(I(SrcReg | Stack, em_push)),
3904 X8(I(DstReg | Stack, em_pop)),
3906 I(ImplicitOps | Stack | No64, em_pusha),
3907 I(ImplicitOps | Stack | No64, em_popa),
3908 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3911 I(SrcImm | Mov | Stack, em_push),
3912 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3913 I(SrcImmByte | Mov | Stack, em_push),
3914 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3915 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3916 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3920 G(ByteOp | DstMem | SrcImm, group1),
3921 G(DstMem | SrcImm, group1),
3922 G(ByteOp | DstMem | SrcImm | No64, group1),
3923 G(DstMem | SrcImmByte, group1),
3924 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3925 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3927 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3928 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3929 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3930 D(ModRM | SrcMem | NoAccess | DstReg),
3931 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3934 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3936 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3937 I(SrcImmFAddr | No64, em_call_far), N,
3938 II(ImplicitOps | Stack, em_pushf, pushf),
3939 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3941 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3942 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3943 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3944 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3946 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3947 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3948 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3949 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3951 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3953 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3955 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3956 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3957 I(ImplicitOps | Stack, em_ret),
3958 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3959 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3960 G(ByteOp, group11), G(0, group11),
3962 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3963 N, I(ImplicitOps | Stack, em_ret_far),
3964 D(ImplicitOps), DI(SrcImmByte, intn),
3965 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3967 G(Src2One | ByteOp, group2), G(Src2One, group2),
3968 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3969 I(DstAcc | SrcImmUByte | No64, em_aam),
3970 I(DstAcc | SrcImmUByte | No64, em_aad),
3971 F(DstAcc | ByteOp | No64, em_salc),
3972 I(DstAcc | SrcXLat | ByteOp, em_mov),
3974 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3976 X3(I(SrcImmByte, em_loop)),
3977 I(SrcImmByte, em_jcxz),
3978 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3979 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3981 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3982 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3983 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3984 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3986 N, DI(ImplicitOps, icebp), N, N,
3987 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3988 G(ByteOp, group3), G(0, group3),
3990 D(ImplicitOps), D(ImplicitOps),
3991 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3992 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3995 static const struct opcode twobyte_table[256] = {
3997 G(0, group6), GD(0, &group7), N, N,
3998 N, I(ImplicitOps | VendorSpecific, em_syscall),
3999 II(ImplicitOps | Priv, em_clts, clts), N,
4000 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4001 N, D(ImplicitOps | ModRM), N, N,
4003 N, N, N, N, N, N, N, N,
4004 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
4006 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4007 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4008 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4009 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4011 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4014 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4015 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4016 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4017 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4018 I(ImplicitOps | VendorSpecific, em_sysenter),
4019 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4021 N, N, N, N, N, N, N, N,
4023 X16(D(DstReg | SrcMem | ModRM | Mov)),
4025 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4030 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4035 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4039 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4041 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4042 II(ImplicitOps, em_cpuid, cpuid),
4043 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4044 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4045 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4047 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4048 DI(ImplicitOps, rsm),
4049 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4050 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4051 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4052 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4054 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4055 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4056 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4057 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4058 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4059 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4063 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4064 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4065 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4067 D2bv(DstMem | SrcReg | ModRM | Lock),
4068 N, D(DstMem | SrcReg | ModRM | Mov),
4069 N, N, N, GD(0, &group9),
4071 X8(I(DstReg, em_bswap)),
4073 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4075 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4077 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4094 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4098 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4104 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4105 unsigned size, bool sign_extension)
4107 int rc = X86EMUL_CONTINUE;
4111 op->addr.mem.ea = ctxt->_eip;
4112 /* NB. Immediates are sign-extended as necessary. */
4113 switch (op->bytes) {
4115 op->val = insn_fetch(s8, ctxt);
4118 op->val = insn_fetch(s16, ctxt);
4121 op->val = insn_fetch(s32, ctxt);
4124 op->val = insn_fetch(s64, ctxt);
4127 if (!sign_extension) {
4128 switch (op->bytes) {
4136 op->val &= 0xffffffff;
4144 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4147 int rc = X86EMUL_CONTINUE;
4151 decode_register_operand(ctxt, op);
4154 rc = decode_imm(ctxt, op, 1, false);
4157 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4161 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4162 fetch_bit_operand(ctxt);
4163 op->orig_val = op->val;
4166 ctxt->memop.bytes = 8;
4170 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4171 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4172 fetch_register_operand(op);
4173 op->orig_val = op->val;
4177 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4179 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4180 op->addr.mem.seg = VCPU_SREG_ES;
4187 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4188 fetch_register_operand(op);
4192 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4195 rc = decode_imm(ctxt, op, 1, true);
4202 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4205 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4208 ctxt->memop.bytes = 1;
4209 if (ctxt->memop.type == OP_REG) {
4210 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4211 fetch_register_operand(&ctxt->memop);
4215 ctxt->memop.bytes = 2;
4218 ctxt->memop.bytes = 4;
4221 rc = decode_imm(ctxt, op, 2, false);
4224 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4228 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4230 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4231 op->addr.mem.seg = seg_override(ctxt);
4237 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4239 register_address(ctxt,
4240 reg_read(ctxt, VCPU_REGS_RBX) +
4241 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4242 op->addr.mem.seg = seg_override(ctxt);
4247 op->addr.mem.ea = ctxt->_eip;
4248 op->bytes = ctxt->op_bytes + 2;
4249 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4252 ctxt->memop.bytes = ctxt->op_bytes + 2;
4255 op->val = VCPU_SREG_ES;
4258 op->val = VCPU_SREG_CS;
4261 op->val = VCPU_SREG_SS;
4264 op->val = VCPU_SREG_DS;
4267 op->val = VCPU_SREG_FS;
4270 op->val = VCPU_SREG_GS;
4273 /* Special instructions do their own operand decoding. */
4275 op->type = OP_NONE; /* Disable writeback. */
4283 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4285 int rc = X86EMUL_CONTINUE;
4286 int mode = ctxt->mode;
4287 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4288 bool op_prefix = false;
4289 struct opcode opcode;
4291 ctxt->memop.type = OP_NONE;
4292 ctxt->memopp = NULL;
4293 ctxt->_eip = ctxt->eip;
4294 ctxt->fetch.start = ctxt->_eip;
4295 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4297 memcpy(ctxt->fetch.data, insn, insn_len);
4300 case X86EMUL_MODE_REAL:
4301 case X86EMUL_MODE_VM86:
4302 case X86EMUL_MODE_PROT16:
4303 def_op_bytes = def_ad_bytes = 2;
4305 case X86EMUL_MODE_PROT32:
4306 def_op_bytes = def_ad_bytes = 4;
4308 #ifdef CONFIG_X86_64
4309 case X86EMUL_MODE_PROT64:
4315 return EMULATION_FAILED;
4318 ctxt->op_bytes = def_op_bytes;
4319 ctxt->ad_bytes = def_ad_bytes;
4321 /* Legacy prefixes. */
4323 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4324 case 0x66: /* operand-size override */
4326 /* switch between 2/4 bytes */
4327 ctxt->op_bytes = def_op_bytes ^ 6;
4329 case 0x67: /* address-size override */
4330 if (mode == X86EMUL_MODE_PROT64)
4331 /* switch between 4/8 bytes */
4332 ctxt->ad_bytes = def_ad_bytes ^ 12;
4334 /* switch between 2/4 bytes */
4335 ctxt->ad_bytes = def_ad_bytes ^ 6;
4337 case 0x26: /* ES override */
4338 case 0x2e: /* CS override */
4339 case 0x36: /* SS override */
4340 case 0x3e: /* DS override */
4341 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4343 case 0x64: /* FS override */
4344 case 0x65: /* GS override */
4345 set_seg_override(ctxt, ctxt->b & 7);
4347 case 0x40 ... 0x4f: /* REX */
4348 if (mode != X86EMUL_MODE_PROT64)
4350 ctxt->rex_prefix = ctxt->b;
4352 case 0xf0: /* LOCK */
4353 ctxt->lock_prefix = 1;
4355 case 0xf2: /* REPNE/REPNZ */
4356 case 0xf3: /* REP/REPE/REPZ */
4357 ctxt->rep_prefix = ctxt->b;
4363 /* Any legacy prefix after a REX prefix nullifies its effect. */
4365 ctxt->rex_prefix = 0;
4371 if (ctxt->rex_prefix & 8)
4372 ctxt->op_bytes = 8; /* REX.W */
4374 /* Opcode byte(s). */
4375 opcode = opcode_table[ctxt->b];
4376 /* Two-byte opcode? */
4377 if (ctxt->b == 0x0f) {
4379 ctxt->b = insn_fetch(u8, ctxt);
4380 opcode = twobyte_table[ctxt->b];
4382 ctxt->d = opcode.flags;
4384 if (ctxt->d & ModRM)
4385 ctxt->modrm = insn_fetch(u8, ctxt);
4387 while (ctxt->d & GroupMask) {
4388 switch (ctxt->d & GroupMask) {
4390 goffset = (ctxt->modrm >> 3) & 7;
4391 opcode = opcode.u.group[goffset];
4394 goffset = (ctxt->modrm >> 3) & 7;
4395 if ((ctxt->modrm >> 6) == 3)
4396 opcode = opcode.u.gdual->mod3[goffset];
4398 opcode = opcode.u.gdual->mod012[goffset];
4401 goffset = ctxt->modrm & 7;
4402 opcode = opcode.u.group[goffset];
4405 if (ctxt->rep_prefix && op_prefix)
4406 return EMULATION_FAILED;
4407 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4408 switch (simd_prefix) {
4409 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4410 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4411 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4412 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4416 if (ctxt->modrm > 0xbf)
4417 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4419 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4422 return EMULATION_FAILED;
4425 ctxt->d &= ~(u64)GroupMask;
4426 ctxt->d |= opcode.flags;
4429 ctxt->execute = opcode.u.execute;
4430 ctxt->check_perm = opcode.check_perm;
4431 ctxt->intercept = opcode.intercept;
4434 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4435 return EMULATION_FAILED;
4437 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4438 return EMULATION_FAILED;
4440 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4443 if (ctxt->d & Op3264) {
4444 if (mode == X86EMUL_MODE_PROT64)
4451 ctxt->op_bytes = 16;
4452 else if (ctxt->d & Mmx)
4455 /* ModRM and SIB bytes. */
4456 if (ctxt->d & ModRM) {
4457 rc = decode_modrm(ctxt, &ctxt->memop);
4458 if (!ctxt->has_seg_override)
4459 set_seg_override(ctxt, ctxt->modrm_seg);
4460 } else if (ctxt->d & MemAbs)
4461 rc = decode_abs(ctxt, &ctxt->memop);
4462 if (rc != X86EMUL_CONTINUE)
4465 if (!ctxt->has_seg_override)
4466 set_seg_override(ctxt, VCPU_SREG_DS);
4468 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4470 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4471 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4474 * Decode and fetch the source operand: register, memory
4477 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4478 if (rc != X86EMUL_CONTINUE)
4482 * Decode and fetch the second source operand: register, memory
4485 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4486 if (rc != X86EMUL_CONTINUE)
4489 /* Decode and fetch the destination operand: register or memory. */
4490 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4493 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4494 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4496 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4499 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4501 return ctxt->d & PageTable;
4504 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4506 /* The second termination condition only applies for REPE
4507 * and REPNE. Test if the repeat string operation prefix is
4508 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4509 * corresponding termination condition according to:
4510 * - if REPE/REPZ and ZF = 0 then done
4511 * - if REPNE/REPNZ and ZF = 1 then done
4513 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4514 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4515 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4516 ((ctxt->eflags & EFLG_ZF) == 0))
4517 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4518 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4524 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4528 ctxt->ops->get_fpu(ctxt);
4529 asm volatile("1: fwait \n\t"
4531 ".pushsection .fixup,\"ax\" \n\t"
4533 "movb $1, %[fault] \n\t"
4536 _ASM_EXTABLE(1b, 3b)
4537 : [fault]"+qm"(fault));
4538 ctxt->ops->put_fpu(ctxt);
4540 if (unlikely(fault))
4541 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4543 return X86EMUL_CONTINUE;
4546 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4549 if (op->type == OP_MM)
4550 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4553 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4555 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4556 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4557 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4558 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4559 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4560 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4561 return X86EMUL_CONTINUE;
4564 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4566 const struct x86_emulate_ops *ops = ctxt->ops;
4567 int rc = X86EMUL_CONTINUE;
4568 int saved_dst_type = ctxt->dst.type;
4570 ctxt->mem_read.pos = 0;
4572 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4573 (ctxt->d & Undefined)) {
4574 rc = emulate_ud(ctxt);
4578 /* LOCK prefix is allowed only with some instructions */
4579 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4580 rc = emulate_ud(ctxt);
4584 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4585 rc = emulate_ud(ctxt);
4589 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4590 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4591 rc = emulate_ud(ctxt);
4595 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4596 rc = emulate_nm(ctxt);
4600 if (ctxt->d & Mmx) {
4601 rc = flush_pending_x87_faults(ctxt);
4602 if (rc != X86EMUL_CONTINUE)
4605 * Now that we know the fpu is exception safe, we can fetch
4608 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4609 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4610 if (!(ctxt->d & Mov))
4611 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4614 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4615 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4616 X86_ICPT_PRE_EXCEPT);
4617 if (rc != X86EMUL_CONTINUE)
4621 /* Privileged instruction can be executed only in CPL=0 */
4622 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4623 rc = emulate_gp(ctxt, 0);
4627 /* Instruction can only be executed in protected mode */
4628 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4629 rc = emulate_ud(ctxt);
4633 /* Do instruction specific permission checks */
4634 if (ctxt->check_perm) {
4635 rc = ctxt->check_perm(ctxt);
4636 if (rc != X86EMUL_CONTINUE)
4640 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4641 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4642 X86_ICPT_POST_EXCEPT);
4643 if (rc != X86EMUL_CONTINUE)
4647 if (ctxt->rep_prefix && (ctxt->d & String)) {
4648 /* All REP prefixes have the same first termination condition */
4649 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4650 ctxt->eip = ctxt->_eip;
4655 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4656 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4657 ctxt->src.valptr, ctxt->src.bytes);
4658 if (rc != X86EMUL_CONTINUE)
4660 ctxt->src.orig_val64 = ctxt->src.val64;
4663 if (ctxt->src2.type == OP_MEM) {
4664 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4665 &ctxt->src2.val, ctxt->src2.bytes);
4666 if (rc != X86EMUL_CONTINUE)
4670 if ((ctxt->d & DstMask) == ImplicitOps)
4674 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4675 /* optimisation - avoid slow emulated read if Mov */
4676 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4677 &ctxt->dst.val, ctxt->dst.bytes);
4678 if (rc != X86EMUL_CONTINUE)
4681 ctxt->dst.orig_val = ctxt->dst.val;
4685 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4686 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4687 X86_ICPT_POST_MEMACCESS);
4688 if (rc != X86EMUL_CONTINUE)
4692 if (ctxt->execute) {
4693 if (ctxt->d & Fastop) {
4694 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4695 rc = fastop(ctxt, fop);
4696 if (rc != X86EMUL_CONTINUE)
4700 rc = ctxt->execute(ctxt);
4701 if (rc != X86EMUL_CONTINUE)
4710 case 0x63: /* movsxd */
4711 if (ctxt->mode != X86EMUL_MODE_PROT64)
4712 goto cannot_emulate;
4713 ctxt->dst.val = (s32) ctxt->src.val;
4715 case 0x70 ... 0x7f: /* jcc (short) */
4716 if (test_cc(ctxt->b, ctxt->eflags))
4717 jmp_rel(ctxt, ctxt->src.val);
4719 case 0x8d: /* lea r16/r32, m */
4720 ctxt->dst.val = ctxt->src.addr.mem.ea;
4722 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4723 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4727 case 0x98: /* cbw/cwde/cdqe */
4728 switch (ctxt->op_bytes) {
4729 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4730 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4731 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4734 case 0xcc: /* int3 */
4735 rc = emulate_int(ctxt, 3);
4737 case 0xcd: /* int n */
4738 rc = emulate_int(ctxt, ctxt->src.val);
4740 case 0xce: /* into */
4741 if (ctxt->eflags & EFLG_OF)
4742 rc = emulate_int(ctxt, 4);
4744 case 0xe9: /* jmp rel */
4745 case 0xeb: /* jmp rel short */
4746 jmp_rel(ctxt, ctxt->src.val);
4747 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4749 case 0xf4: /* hlt */
4750 ctxt->ops->halt(ctxt);
4752 case 0xf5: /* cmc */
4753 /* complement carry flag from eflags reg */
4754 ctxt->eflags ^= EFLG_CF;
4756 case 0xf8: /* clc */
4757 ctxt->eflags &= ~EFLG_CF;
4759 case 0xf9: /* stc */
4760 ctxt->eflags |= EFLG_CF;
4762 case 0xfc: /* cld */
4763 ctxt->eflags &= ~EFLG_DF;
4765 case 0xfd: /* std */
4766 ctxt->eflags |= EFLG_DF;
4769 goto cannot_emulate;
4772 if (rc != X86EMUL_CONTINUE)
4776 rc = writeback(ctxt);
4777 if (rc != X86EMUL_CONTINUE)
4781 * restore dst type in case the decoding will be reused
4782 * (happens for string instruction )
4784 ctxt->dst.type = saved_dst_type;
4786 if ((ctxt->d & SrcMask) == SrcSI)
4787 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4789 if ((ctxt->d & DstMask) == DstDI)
4790 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4792 if (ctxt->rep_prefix && (ctxt->d & String)) {
4794 struct read_cache *r = &ctxt->io_read;
4795 if ((ctxt->d & SrcMask) == SrcSI)
4796 count = ctxt->src.count;
4798 count = ctxt->dst.count;
4799 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4802 if (!string_insn_completed(ctxt)) {
4804 * Re-enter guest when pio read ahead buffer is empty
4805 * or, if it is not used, after each 1024 iteration.
4807 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4808 (r->end == 0 || r->end != r->pos)) {
4810 * Reset read cache. Usually happens before
4811 * decode, but since instruction is restarted
4812 * we have to do it here.
4814 ctxt->mem_read.end = 0;
4815 writeback_registers(ctxt);
4816 return EMULATION_RESTART;
4818 goto done; /* skip rip writeback */
4822 ctxt->eip = ctxt->_eip;
4825 if (rc == X86EMUL_PROPAGATE_FAULT)
4826 ctxt->have_exception = true;
4827 if (rc == X86EMUL_INTERCEPTED)
4828 return EMULATION_INTERCEPTED;
4830 if (rc == X86EMUL_CONTINUE)
4831 writeback_registers(ctxt);
4833 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4837 case 0x09: /* wbinvd */
4838 (ctxt->ops->wbinvd)(ctxt);
4840 case 0x08: /* invd */
4841 case 0x0d: /* GrpP (prefetch) */
4842 case 0x18: /* Grp16 (prefetch/nop) */
4843 case 0x1f: /* nop */
4845 case 0x20: /* mov cr, reg */
4846 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4848 case 0x21: /* mov from dr to reg */
4849 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4851 case 0x40 ... 0x4f: /* cmov */
4852 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4853 if (!test_cc(ctxt->b, ctxt->eflags))
4854 ctxt->dst.type = OP_NONE; /* no writeback */
4856 case 0x80 ... 0x8f: /* jnz rel, etc*/
4857 if (test_cc(ctxt->b, ctxt->eflags))
4858 jmp_rel(ctxt, ctxt->src.val);
4860 case 0x90 ... 0x9f: /* setcc r/m8 */
4861 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4863 case 0xae: /* clflush */
4865 case 0xb6 ... 0xb7: /* movzx */
4866 ctxt->dst.bytes = ctxt->op_bytes;
4867 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4868 : (u16) ctxt->src.val;
4870 case 0xbe ... 0xbf: /* movsx */
4871 ctxt->dst.bytes = ctxt->op_bytes;
4872 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4873 (s16) ctxt->src.val;
4875 case 0xc0 ... 0xc1: /* xadd */
4876 fastop(ctxt, em_add);
4877 /* Write back the register source. */
4878 ctxt->src.val = ctxt->dst.orig_val;
4879 write_register_operand(&ctxt->src);
4881 case 0xc3: /* movnti */
4882 ctxt->dst.bytes = ctxt->op_bytes;
4883 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4884 (u64) ctxt->src.val;
4887 goto cannot_emulate;
4890 if (rc != X86EMUL_CONTINUE)
4896 return EMULATION_FAILED;
4899 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4901 invalidate_registers(ctxt);
4904 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4906 writeback_registers(ctxt);