1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstDX (8<<1) /* Destination is in DX register */
51 #define DstMask (0xf<<1)
52 /* Source operand type. */
53 #define SrcNone (0<<5) /* No source operand. */
54 #define SrcReg (1<<5) /* Register operand. */
55 #define SrcMem (2<<5) /* Memory operand. */
56 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
57 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
58 #define SrcImm (5<<5) /* Immediate operand. */
59 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
60 #define SrcOne (7<<5) /* Implied '1' */
61 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
62 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
63 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
64 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
65 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
66 #define SrcAcc (0xd<<5) /* Source Accumulator */
67 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
68 #define SrcDX (0xf<<5) /* Source is in DX register */
69 #define SrcMask (0xf<<5)
70 /* Generic ModRM decode. */
72 /* Destination is only written; never read. */
75 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
76 #define String (1<<13) /* String instruction (rep capable) */
77 #define Stack (1<<14) /* Stack instruction (push/pop) */
78 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
79 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
80 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
81 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
82 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
83 #define Sse (1<<18) /* SSE Vector instruction */
85 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
86 #define VendorSpecific (1<<22) /* Vendor specific instruction */
87 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89 #define Undefined (1<<25) /* No Such Instruction */
90 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
91 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
93 /* Source 2 operand type */
94 #define Src2None (0<<29)
95 #define Src2CL (1<<29)
96 #define Src2ImmByte (2<<29)
97 #define Src2One (3<<29)
98 #define Src2Imm (4<<29)
99 #define Src2Mask (7<<29)
101 #define X2(x...) x, x
102 #define X3(x...) X2(x), x
103 #define X4(x...) X2(x), X2(x)
104 #define X5(x...) X4(x), x
105 #define X6(x...) X4(x), X2(x)
106 #define X7(x...) X4(x), X3(x)
107 #define X8(x...) X4(x), X4(x)
108 #define X16(x...) X8(x), X8(x)
114 int (*execute)(struct x86_emulate_ctxt *ctxt);
115 struct opcode *group;
116 struct group_dual *gdual;
117 struct gprefix *gprefix;
119 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
123 struct opcode mod012[8];
124 struct opcode mod3[8];
128 struct opcode pfx_no;
129 struct opcode pfx_66;
130 struct opcode pfx_f2;
131 struct opcode pfx_f3;
134 /* EFLAGS bit definitions. */
135 #define EFLG_ID (1<<21)
136 #define EFLG_VIP (1<<20)
137 #define EFLG_VIF (1<<19)
138 #define EFLG_AC (1<<18)
139 #define EFLG_VM (1<<17)
140 #define EFLG_RF (1<<16)
141 #define EFLG_IOPL (3<<12)
142 #define EFLG_NT (1<<14)
143 #define EFLG_OF (1<<11)
144 #define EFLG_DF (1<<10)
145 #define EFLG_IF (1<<9)
146 #define EFLG_TF (1<<8)
147 #define EFLG_SF (1<<7)
148 #define EFLG_ZF (1<<6)
149 #define EFLG_AF (1<<4)
150 #define EFLG_PF (1<<2)
151 #define EFLG_CF (1<<0)
153 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
154 #define EFLG_RESERVED_ONE_MASK 2
157 * Instruction emulation:
158 * Most instructions are emulated directly via a fragment of inline assembly
159 * code. This allows us to save/restore EFLAGS and thus very easily pick up
160 * any modified flags.
163 #if defined(CONFIG_X86_64)
164 #define _LO32 "k" /* force 32-bit operand */
165 #define _STK "%%rsp" /* stack pointer */
166 #elif defined(__i386__)
167 #define _LO32 "" /* force 32-bit operand */
168 #define _STK "%%esp" /* stack pointer */
172 * These EFLAGS bits are restored from saved value during emulation, and
173 * any changes are written back to the saved value after emulation.
175 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
177 /* Before executing instruction: restore necessary bits in EFLAGS. */
178 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
179 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
180 "movl %"_sav",%"_LO32 _tmp"; " \
183 "movl %"_msk",%"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
186 "notl %"_LO32 _tmp"; " \
187 "andl %"_LO32 _tmp",("_STK"); " \
188 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
190 "orl %"_LO32 _tmp",("_STK"); " \
194 /* After executing instruction: write-back necessary bits in EFLAGS. */
195 #define _POST_EFLAGS(_sav, _msk, _tmp) \
196 /* _sav |= EFLAGS & _msk; */ \
199 "andl %"_msk",%"_LO32 _tmp"; " \
200 "orl %"_LO32 _tmp",%"_sav"; "
208 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
210 __asm__ __volatile__ ( \
211 _PRE_EFLAGS("0", "4", "2") \
212 _op _suffix " %"_x"3,%1; " \
213 _POST_EFLAGS("0", "4", "2") \
214 : "=m" ((ctxt)->eflags), \
215 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
217 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
221 /* Raw emulation: instruction has two explicit operands. */
222 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
224 unsigned long _tmp; \
226 switch ((ctxt)->dst.bytes) { \
228 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
231 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
234 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
239 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
241 unsigned long _tmp; \
242 switch ((ctxt)->dst.bytes) { \
244 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
247 __emulate_2op_nobyte(ctxt, _op, \
248 _wx, _wy, _lx, _ly, _qx, _qy); \
253 /* Source operand is byte-sized and may be restricted to just %cl. */
254 #define emulate_2op_SrcB(ctxt, _op) \
255 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
257 /* Source operand is byte, word, long or quad sized. */
258 #define emulate_2op_SrcV(ctxt, _op) \
259 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
261 /* Source operand is word, long or quad sized. */
262 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
263 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
265 /* Instruction has three operands and one operand is stored in ECX register */
266 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
268 unsigned long _tmp; \
269 _type _clv = (ctxt)->src2.val; \
270 _type _srcv = (ctxt)->src.val; \
271 _type _dstv = (ctxt)->dst.val; \
273 __asm__ __volatile__ ( \
274 _PRE_EFLAGS("0", "5", "2") \
275 _op _suffix " %4,%1 \n" \
276 _POST_EFLAGS("0", "5", "2") \
277 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
278 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
281 (ctxt)->src2.val = (unsigned long) _clv; \
282 (ctxt)->src2.val = (unsigned long) _srcv; \
283 (ctxt)->dst.val = (unsigned long) _dstv; \
286 #define emulate_2op_cl(ctxt, _op) \
288 switch ((ctxt)->dst.bytes) { \
290 __emulate_2op_cl(ctxt, _op, "w", u16); \
293 __emulate_2op_cl(ctxt, _op, "l", u32); \
296 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
301 #define __emulate_1op(ctxt, _op, _suffix) \
303 unsigned long _tmp; \
305 __asm__ __volatile__ ( \
306 _PRE_EFLAGS("0", "3", "2") \
307 _op _suffix " %1; " \
308 _POST_EFLAGS("0", "3", "2") \
309 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
311 : "i" (EFLAGS_MASK)); \
314 /* Instruction has only one explicit operand (no source operand). */
315 #define emulate_1op(ctxt, _op) \
317 switch ((ctxt)->dst.bytes) { \
318 case 1: __emulate_1op(ctxt, _op, "b"); break; \
319 case 2: __emulate_1op(ctxt, _op, "w"); break; \
320 case 4: __emulate_1op(ctxt, _op, "l"); break; \
321 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
325 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
327 unsigned long _tmp; \
328 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \
329 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \
331 __asm__ __volatile__ ( \
332 _PRE_EFLAGS("0", "5", "1") \
334 _op _suffix " %6; " \
336 _POST_EFLAGS("0", "5", "1") \
337 ".pushsection .fixup,\"ax\" \n\t" \
338 "3: movb $1, %4 \n\t" \
341 _ASM_EXTABLE(1b, 3b) \
342 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
343 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
344 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val), \
345 "a" (*rax), "d" (*rdx)); \
348 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
349 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
351 switch((ctxt)->src.bytes) { \
353 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
356 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
359 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
362 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
367 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
368 enum x86_intercept intercept,
369 enum x86_intercept_stage stage)
371 struct x86_instruction_info info = {
372 .intercept = intercept,
373 .rep_prefix = ctxt->rep_prefix,
374 .modrm_mod = ctxt->modrm_mod,
375 .modrm_reg = ctxt->modrm_reg,
376 .modrm_rm = ctxt->modrm_rm,
377 .src_val = ctxt->src.val64,
378 .src_bytes = ctxt->src.bytes,
379 .dst_bytes = ctxt->dst.bytes,
380 .ad_bytes = ctxt->ad_bytes,
381 .next_rip = ctxt->eip,
384 return ctxt->ops->intercept(ctxt, &info, stage);
387 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
389 return (1UL << (ctxt->ad_bytes << 3)) - 1;
392 /* Access/update address held in a register, based on addressing mode. */
393 static inline unsigned long
394 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
396 if (ctxt->ad_bytes == sizeof(unsigned long))
399 return reg & ad_mask(ctxt);
402 static inline unsigned long
403 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
405 return address_mask(ctxt, reg);
409 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
411 if (ctxt->ad_bytes == sizeof(unsigned long))
414 *reg = (*reg & ~ad_mask(ctxt)) | ((*reg + inc) & ad_mask(ctxt));
417 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
419 register_address_increment(ctxt, &ctxt->_eip, rel);
422 static u32 desc_limit_scaled(struct desc_struct *desc)
424 u32 limit = get_desc_limit(desc);
426 return desc->g ? (limit << 12) | 0xfff : limit;
429 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
431 ctxt->has_seg_override = true;
432 ctxt->seg_override = seg;
435 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
437 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
440 return ctxt->ops->get_cached_segment_base(ctxt, seg);
443 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
445 if (!ctxt->has_seg_override)
448 return ctxt->seg_override;
451 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
452 u32 error, bool valid)
454 ctxt->exception.vector = vec;
455 ctxt->exception.error_code = error;
456 ctxt->exception.error_code_valid = valid;
457 return X86EMUL_PROPAGATE_FAULT;
460 static int emulate_db(struct x86_emulate_ctxt *ctxt)
462 return emulate_exception(ctxt, DB_VECTOR, 0, false);
465 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
467 return emulate_exception(ctxt, GP_VECTOR, err, true);
470 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
472 return emulate_exception(ctxt, SS_VECTOR, err, true);
475 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
477 return emulate_exception(ctxt, UD_VECTOR, 0, false);
480 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
482 return emulate_exception(ctxt, TS_VECTOR, err, true);
485 static int emulate_de(struct x86_emulate_ctxt *ctxt)
487 return emulate_exception(ctxt, DE_VECTOR, 0, false);
490 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
492 return emulate_exception(ctxt, NM_VECTOR, 0, false);
495 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
498 struct desc_struct desc;
500 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
504 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
509 struct desc_struct desc;
511 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
512 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
515 static int __linearize(struct x86_emulate_ctxt *ctxt,
516 struct segmented_address addr,
517 unsigned size, bool write, bool fetch,
520 struct desc_struct desc;
527 la = seg_base(ctxt, addr.seg) + addr.ea;
528 switch (ctxt->mode) {
529 case X86EMUL_MODE_REAL:
531 case X86EMUL_MODE_PROT64:
532 if (((signed long)la << 16) >> 16 != la)
533 return emulate_gp(ctxt, 0);
536 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
540 /* code segment or read-only data segment */
541 if (((desc.type & 8) || !(desc.type & 2)) && write)
543 /* unreadable code segment */
544 if (!fetch && (desc.type & 8) && !(desc.type & 2))
546 lim = desc_limit_scaled(&desc);
547 if ((desc.type & 8) || !(desc.type & 4)) {
548 /* expand-up segment */
549 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
552 /* exapand-down segment */
553 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
555 lim = desc.d ? 0xffffffff : 0xffff;
556 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
559 cpl = ctxt->ops->cpl(ctxt);
562 if (!(desc.type & 8)) {
566 } else if ((desc.type & 8) && !(desc.type & 4)) {
567 /* nonconforming code segment */
570 } else if ((desc.type & 8) && (desc.type & 4)) {
571 /* conforming code segment */
577 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
580 return X86EMUL_CONTINUE;
582 if (addr.seg == VCPU_SREG_SS)
583 return emulate_ss(ctxt, addr.seg);
585 return emulate_gp(ctxt, addr.seg);
588 static int linearize(struct x86_emulate_ctxt *ctxt,
589 struct segmented_address addr,
590 unsigned size, bool write,
593 return __linearize(ctxt, addr, size, write, false, linear);
597 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
598 struct segmented_address addr,
605 rc = linearize(ctxt, addr, size, false, &linear);
606 if (rc != X86EMUL_CONTINUE)
608 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
612 * Fetch the next byte of the instruction being emulated which is pointed to
613 * by ctxt->_eip, then increment ctxt->_eip.
615 * Also prefetch the remaining bytes of the instruction without crossing page
616 * boundary if they are not in fetch_cache yet.
618 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
620 struct fetch_cache *fc = &ctxt->fetch;
624 if (ctxt->_eip == fc->end) {
625 unsigned long linear;
626 struct segmented_address addr = { .seg = VCPU_SREG_CS,
628 cur_size = fc->end - fc->start;
629 size = min(15UL - cur_size,
630 PAGE_SIZE - offset_in_page(ctxt->_eip));
631 rc = __linearize(ctxt, addr, size, false, true, &linear);
632 if (unlikely(rc != X86EMUL_CONTINUE))
634 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
635 size, &ctxt->exception);
636 if (unlikely(rc != X86EMUL_CONTINUE))
640 *dest = fc->data[ctxt->_eip - fc->start];
642 return X86EMUL_CONTINUE;
645 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
646 void *dest, unsigned size)
650 /* x86 instructions are limited to 15 bytes. */
651 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
652 return X86EMUL_UNHANDLEABLE;
654 rc = do_insn_fetch_byte(ctxt, dest++);
655 if (rc != X86EMUL_CONTINUE)
658 return X86EMUL_CONTINUE;
661 /* Fetch next part of the instruction being emulated. */
662 #define insn_fetch(_type, _ctxt) \
663 ({ unsigned long _x; \
664 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
665 if (rc != X86EMUL_CONTINUE) \
670 #define insn_fetch_arr(_arr, _size, _ctxt) \
671 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
672 if (rc != X86EMUL_CONTINUE) \
677 * Given the 'reg' portion of a ModRM byte, and a register block, return a
678 * pointer into the block that addresses the relevant register.
679 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
681 static void *decode_register(u8 modrm_reg, unsigned long *regs,
686 p = ®s[modrm_reg];
687 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
688 p = (unsigned char *)®s[modrm_reg & 3] + 1;
692 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
693 struct segmented_address addr,
694 u16 *size, unsigned long *address, int op_bytes)
701 rc = segmented_read_std(ctxt, addr, size, 2);
702 if (rc != X86EMUL_CONTINUE)
705 rc = segmented_read_std(ctxt, addr, address, op_bytes);
709 static int test_cc(unsigned int condition, unsigned int flags)
713 switch ((condition & 15) >> 1) {
715 rc |= (flags & EFLG_OF);
717 case 1: /* b/c/nae */
718 rc |= (flags & EFLG_CF);
721 rc |= (flags & EFLG_ZF);
724 rc |= (flags & (EFLG_CF|EFLG_ZF));
727 rc |= (flags & EFLG_SF);
730 rc |= (flags & EFLG_PF);
733 rc |= (flags & EFLG_ZF);
736 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
740 /* Odd condition identifiers (lsb == 1) have inverted sense. */
741 return (!!rc ^ (condition & 1));
744 static void fetch_register_operand(struct operand *op)
748 op->val = *(u8 *)op->addr.reg;
751 op->val = *(u16 *)op->addr.reg;
754 op->val = *(u32 *)op->addr.reg;
757 op->val = *(u64 *)op->addr.reg;
762 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
764 ctxt->ops->get_fpu(ctxt);
766 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
767 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
768 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
769 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
770 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
771 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
772 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
773 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
775 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
776 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
777 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
778 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
779 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
780 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
781 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
782 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
786 ctxt->ops->put_fpu(ctxt);
789 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
792 ctxt->ops->get_fpu(ctxt);
794 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
795 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
796 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
797 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
798 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
799 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
800 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
801 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
803 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
804 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
805 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
806 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
807 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
808 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
809 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
810 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
814 ctxt->ops->put_fpu(ctxt);
817 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
821 unsigned reg = ctxt->modrm_reg;
822 int highbyte_regs = ctxt->rex_prefix == 0;
824 if (!(ctxt->d & ModRM))
825 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
831 read_sse_reg(ctxt, &op->vec_val, reg);
836 if ((ctxt->d & ByteOp) && !inhibit_bytereg) {
837 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
840 op->addr.reg = decode_register(reg, ctxt->regs, 0);
841 op->bytes = ctxt->op_bytes;
843 fetch_register_operand(op);
844 op->orig_val = op->val;
847 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
851 int index_reg = 0, base_reg = 0, scale;
852 int rc = X86EMUL_CONTINUE;
855 if (ctxt->rex_prefix) {
856 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
857 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
858 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
861 ctxt->modrm = insn_fetch(u8, ctxt);
862 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
863 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
864 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
865 ctxt->modrm_seg = VCPU_SREG_DS;
867 if (ctxt->modrm_mod == 3) {
869 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
870 op->addr.reg = decode_register(ctxt->modrm_rm,
871 ctxt->regs, ctxt->d & ByteOp);
875 op->addr.xmm = ctxt->modrm_rm;
876 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
879 fetch_register_operand(op);
885 if (ctxt->ad_bytes == 2) {
886 unsigned bx = ctxt->regs[VCPU_REGS_RBX];
887 unsigned bp = ctxt->regs[VCPU_REGS_RBP];
888 unsigned si = ctxt->regs[VCPU_REGS_RSI];
889 unsigned di = ctxt->regs[VCPU_REGS_RDI];
891 /* 16-bit ModR/M decode. */
892 switch (ctxt->modrm_mod) {
894 if (ctxt->modrm_rm == 6)
895 modrm_ea += insn_fetch(u16, ctxt);
898 modrm_ea += insn_fetch(s8, ctxt);
901 modrm_ea += insn_fetch(u16, ctxt);
904 switch (ctxt->modrm_rm) {
924 if (ctxt->modrm_mod != 0)
931 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
932 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
933 ctxt->modrm_seg = VCPU_SREG_SS;
934 modrm_ea = (u16)modrm_ea;
936 /* 32/64-bit ModR/M decode. */
937 if ((ctxt->modrm_rm & 7) == 4) {
938 sib = insn_fetch(u8, ctxt);
939 index_reg |= (sib >> 3) & 7;
943 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
944 modrm_ea += insn_fetch(s32, ctxt);
946 modrm_ea += ctxt->regs[base_reg];
948 modrm_ea += ctxt->regs[index_reg] << scale;
949 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
950 if (ctxt->mode == X86EMUL_MODE_PROT64)
951 ctxt->rip_relative = 1;
953 modrm_ea += ctxt->regs[ctxt->modrm_rm];
954 switch (ctxt->modrm_mod) {
956 if (ctxt->modrm_rm == 5)
957 modrm_ea += insn_fetch(s32, ctxt);
960 modrm_ea += insn_fetch(s8, ctxt);
963 modrm_ea += insn_fetch(s32, ctxt);
967 op->addr.mem.ea = modrm_ea;
972 static int decode_abs(struct x86_emulate_ctxt *ctxt,
975 int rc = X86EMUL_CONTINUE;
978 switch (ctxt->ad_bytes) {
980 op->addr.mem.ea = insn_fetch(u16, ctxt);
983 op->addr.mem.ea = insn_fetch(u32, ctxt);
986 op->addr.mem.ea = insn_fetch(u64, ctxt);
993 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
997 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
998 mask = ~(ctxt->dst.bytes * 8 - 1);
1000 if (ctxt->src.bytes == 2)
1001 sv = (s16)ctxt->src.val & (s16)mask;
1002 else if (ctxt->src.bytes == 4)
1003 sv = (s32)ctxt->src.val & (s32)mask;
1005 ctxt->dst.addr.mem.ea += (sv >> 3);
1008 /* only subword offset */
1009 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1012 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1013 unsigned long addr, void *dest, unsigned size)
1016 struct read_cache *mc = &ctxt->mem_read;
1019 int n = min(size, 8u);
1021 if (mc->pos < mc->end)
1024 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1026 if (rc != X86EMUL_CONTINUE)
1031 memcpy(dest, mc->data + mc->pos, n);
1036 return X86EMUL_CONTINUE;
1039 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1040 struct segmented_address addr,
1047 rc = linearize(ctxt, addr, size, false, &linear);
1048 if (rc != X86EMUL_CONTINUE)
1050 return read_emulated(ctxt, linear, data, size);
1053 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1054 struct segmented_address addr,
1061 rc = linearize(ctxt, addr, size, true, &linear);
1062 if (rc != X86EMUL_CONTINUE)
1064 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1068 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1069 struct segmented_address addr,
1070 const void *orig_data, const void *data,
1076 rc = linearize(ctxt, addr, size, true, &linear);
1077 if (rc != X86EMUL_CONTINUE)
1079 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1080 size, &ctxt->exception);
1083 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1084 unsigned int size, unsigned short port,
1087 struct read_cache *rc = &ctxt->io_read;
1089 if (rc->pos == rc->end) { /* refill pio read ahead */
1090 unsigned int in_page, n;
1091 unsigned int count = ctxt->rep_prefix ?
1092 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
1093 in_page = (ctxt->eflags & EFLG_DF) ?
1094 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
1095 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
1096 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1100 rc->pos = rc->end = 0;
1101 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1106 memcpy(dest, rc->data + rc->pos, size);
1111 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1112 u16 selector, struct desc_ptr *dt)
1114 struct x86_emulate_ops *ops = ctxt->ops;
1116 if (selector & 1 << 2) {
1117 struct desc_struct desc;
1120 memset (dt, 0, sizeof *dt);
1121 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1124 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1125 dt->address = get_desc_base(&desc);
1127 ops->get_gdt(ctxt, dt);
1130 /* allowed just for 8 bytes segments */
1131 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1132 u16 selector, struct desc_struct *desc)
1135 u16 index = selector >> 3;
1138 get_descriptor_table_ptr(ctxt, selector, &dt);
1140 if (dt.size < index * 8 + 7)
1141 return emulate_gp(ctxt, selector & 0xfffc);
1143 addr = dt.address + index * 8;
1144 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1148 /* allowed just for 8 bytes segments */
1149 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1150 u16 selector, struct desc_struct *desc)
1153 u16 index = selector >> 3;
1156 get_descriptor_table_ptr(ctxt, selector, &dt);
1158 if (dt.size < index * 8 + 7)
1159 return emulate_gp(ctxt, selector & 0xfffc);
1161 addr = dt.address + index * 8;
1162 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1166 /* Does not support long mode */
1167 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1168 u16 selector, int seg)
1170 struct desc_struct seg_desc;
1172 unsigned err_vec = GP_VECTOR;
1174 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1177 memset(&seg_desc, 0, sizeof seg_desc);
1179 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1180 || ctxt->mode == X86EMUL_MODE_REAL) {
1181 /* set real mode segment descriptor */
1182 set_desc_base(&seg_desc, selector << 4);
1183 set_desc_limit(&seg_desc, 0xffff);
1190 /* NULL selector is not valid for TR, CS and SS */
1191 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1195 /* TR should be in GDT only */
1196 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1199 if (null_selector) /* for NULL selector skip all following checks */
1202 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1203 if (ret != X86EMUL_CONTINUE)
1206 err_code = selector & 0xfffc;
1207 err_vec = GP_VECTOR;
1209 /* can't load system descriptor into segment selecor */
1210 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1214 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1220 cpl = ctxt->ops->cpl(ctxt);
1225 * segment is not a writable data segment or segment
1226 * selector's RPL != CPL or segment selector's RPL != CPL
1228 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1232 if (!(seg_desc.type & 8))
1235 if (seg_desc.type & 4) {
1241 if (rpl > cpl || dpl != cpl)
1244 /* CS(RPL) <- CPL */
1245 selector = (selector & 0xfffc) | cpl;
1248 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1251 case VCPU_SREG_LDTR:
1252 if (seg_desc.s || seg_desc.type != 2)
1255 default: /* DS, ES, FS, or GS */
1257 * segment is not a data or readable code segment or
1258 * ((segment is a data or nonconforming code segment)
1259 * and (both RPL and CPL > DPL))
1261 if ((seg_desc.type & 0xa) == 0x8 ||
1262 (((seg_desc.type & 0xc) != 0xc) &&
1263 (rpl > dpl && cpl > dpl)))
1269 /* mark segment as accessed */
1271 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1272 if (ret != X86EMUL_CONTINUE)
1276 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1277 return X86EMUL_CONTINUE;
1279 emulate_exception(ctxt, err_vec, err_code, true);
1280 return X86EMUL_PROPAGATE_FAULT;
1283 static void write_register_operand(struct operand *op)
1285 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1286 switch (op->bytes) {
1288 *(u8 *)op->addr.reg = (u8)op->val;
1291 *(u16 *)op->addr.reg = (u16)op->val;
1294 *op->addr.reg = (u32)op->val;
1295 break; /* 64b: zero-extend */
1297 *op->addr.reg = op->val;
1302 static int writeback(struct x86_emulate_ctxt *ctxt)
1306 switch (ctxt->dst.type) {
1308 write_register_operand(&ctxt->dst);
1311 if (ctxt->lock_prefix)
1312 rc = segmented_cmpxchg(ctxt,
1314 &ctxt->dst.orig_val,
1318 rc = segmented_write(ctxt,
1322 if (rc != X86EMUL_CONTINUE)
1326 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1334 return X86EMUL_CONTINUE;
1337 static int em_push(struct x86_emulate_ctxt *ctxt)
1339 struct segmented_address addr;
1341 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], -ctxt->op_bytes);
1342 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1343 addr.seg = VCPU_SREG_SS;
1345 /* Disable writeback. */
1346 ctxt->dst.type = OP_NONE;
1347 return segmented_write(ctxt, addr, &ctxt->src.val, ctxt->op_bytes);
1350 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1351 void *dest, int len)
1354 struct segmented_address addr;
1356 addr.ea = register_address(ctxt, ctxt->regs[VCPU_REGS_RSP]);
1357 addr.seg = VCPU_SREG_SS;
1358 rc = segmented_read(ctxt, addr, dest, len);
1359 if (rc != X86EMUL_CONTINUE)
1362 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], len);
1366 static int em_pop(struct x86_emulate_ctxt *ctxt)
1368 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1371 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1372 void *dest, int len)
1375 unsigned long val, change_mask;
1376 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1377 int cpl = ctxt->ops->cpl(ctxt);
1379 rc = emulate_pop(ctxt, &val, len);
1380 if (rc != X86EMUL_CONTINUE)
1383 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1384 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1386 switch(ctxt->mode) {
1387 case X86EMUL_MODE_PROT64:
1388 case X86EMUL_MODE_PROT32:
1389 case X86EMUL_MODE_PROT16:
1391 change_mask |= EFLG_IOPL;
1393 change_mask |= EFLG_IF;
1395 case X86EMUL_MODE_VM86:
1397 return emulate_gp(ctxt, 0);
1398 change_mask |= EFLG_IF;
1400 default: /* real mode */
1401 change_mask |= (EFLG_IOPL | EFLG_IF);
1405 *(unsigned long *)dest =
1406 (ctxt->eflags & ~change_mask) | (val & change_mask);
1411 static int em_popf(struct x86_emulate_ctxt *ctxt)
1413 ctxt->dst.type = OP_REG;
1414 ctxt->dst.addr.reg = &ctxt->eflags;
1415 ctxt->dst.bytes = ctxt->op_bytes;
1416 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1419 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1421 ctxt->src.val = get_segment_selector(ctxt, seg);
1423 return em_push(ctxt);
1426 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1428 unsigned long selector;
1431 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1432 if (rc != X86EMUL_CONTINUE)
1435 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1439 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1441 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
1442 int rc = X86EMUL_CONTINUE;
1443 int reg = VCPU_REGS_RAX;
1445 while (reg <= VCPU_REGS_RDI) {
1446 (reg == VCPU_REGS_RSP) ?
1447 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
1450 if (rc != X86EMUL_CONTINUE)
1459 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1461 ctxt->src.val = (unsigned long)ctxt->eflags;
1462 return em_push(ctxt);
1465 static int em_popa(struct x86_emulate_ctxt *ctxt)
1467 int rc = X86EMUL_CONTINUE;
1468 int reg = VCPU_REGS_RDI;
1470 while (reg >= VCPU_REGS_RAX) {
1471 if (reg == VCPU_REGS_RSP) {
1472 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP],
1477 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
1478 if (rc != X86EMUL_CONTINUE)
1485 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1487 struct x86_emulate_ops *ops = ctxt->ops;
1494 /* TODO: Add limit checks */
1495 ctxt->src.val = ctxt->eflags;
1497 if (rc != X86EMUL_CONTINUE)
1500 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1502 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1504 if (rc != X86EMUL_CONTINUE)
1507 ctxt->src.val = ctxt->_eip;
1509 if (rc != X86EMUL_CONTINUE)
1512 ops->get_idt(ctxt, &dt);
1514 eip_addr = dt.address + (irq << 2);
1515 cs_addr = dt.address + (irq << 2) + 2;
1517 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1518 if (rc != X86EMUL_CONTINUE)
1521 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1522 if (rc != X86EMUL_CONTINUE)
1525 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1526 if (rc != X86EMUL_CONTINUE)
1534 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1536 switch(ctxt->mode) {
1537 case X86EMUL_MODE_REAL:
1538 return emulate_int_real(ctxt, irq);
1539 case X86EMUL_MODE_VM86:
1540 case X86EMUL_MODE_PROT16:
1541 case X86EMUL_MODE_PROT32:
1542 case X86EMUL_MODE_PROT64:
1544 /* Protected mode interrupts unimplemented yet */
1545 return X86EMUL_UNHANDLEABLE;
1549 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1551 int rc = X86EMUL_CONTINUE;
1552 unsigned long temp_eip = 0;
1553 unsigned long temp_eflags = 0;
1554 unsigned long cs = 0;
1555 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1556 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1557 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1558 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1560 /* TODO: Add stack limit check */
1562 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1564 if (rc != X86EMUL_CONTINUE)
1567 if (temp_eip & ~0xffff)
1568 return emulate_gp(ctxt, 0);
1570 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1572 if (rc != X86EMUL_CONTINUE)
1575 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1577 if (rc != X86EMUL_CONTINUE)
1580 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1582 if (rc != X86EMUL_CONTINUE)
1585 ctxt->_eip = temp_eip;
1588 if (ctxt->op_bytes == 4)
1589 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1590 else if (ctxt->op_bytes == 2) {
1591 ctxt->eflags &= ~0xffff;
1592 ctxt->eflags |= temp_eflags;
1595 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1596 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1601 static int em_iret(struct x86_emulate_ctxt *ctxt)
1603 switch(ctxt->mode) {
1604 case X86EMUL_MODE_REAL:
1605 return emulate_iret_real(ctxt);
1606 case X86EMUL_MODE_VM86:
1607 case X86EMUL_MODE_PROT16:
1608 case X86EMUL_MODE_PROT32:
1609 case X86EMUL_MODE_PROT64:
1611 /* iret from protected mode unimplemented yet */
1612 return X86EMUL_UNHANDLEABLE;
1616 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1621 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1623 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1624 if (rc != X86EMUL_CONTINUE)
1628 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1629 return X86EMUL_CONTINUE;
1632 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1634 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->dst.bytes);
1637 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1639 switch (ctxt->modrm_reg) {
1641 emulate_2op_SrcB(ctxt, "rol");
1644 emulate_2op_SrcB(ctxt, "ror");
1647 emulate_2op_SrcB(ctxt, "rcl");
1650 emulate_2op_SrcB(ctxt, "rcr");
1652 case 4: /* sal/shl */
1653 case 6: /* sal/shl */
1654 emulate_2op_SrcB(ctxt, "sal");
1657 emulate_2op_SrcB(ctxt, "shr");
1660 emulate_2op_SrcB(ctxt, "sar");
1663 return X86EMUL_CONTINUE;
1666 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1670 switch (ctxt->modrm_reg) {
1671 case 0 ... 1: /* test */
1672 emulate_2op_SrcV(ctxt, "test");
1673 /* Disable writeback. */
1674 ctxt->dst.type = OP_NONE;
1677 ctxt->dst.val = ~ctxt->dst.val;
1680 emulate_1op(ctxt, "neg");
1683 emulate_1op_rax_rdx(ctxt, "mul", de);
1686 emulate_1op_rax_rdx(ctxt, "imul", de);
1689 emulate_1op_rax_rdx(ctxt, "div", de);
1692 emulate_1op_rax_rdx(ctxt, "idiv", de);
1695 return X86EMUL_UNHANDLEABLE;
1698 return emulate_de(ctxt);
1699 return X86EMUL_CONTINUE;
1702 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1704 int rc = X86EMUL_CONTINUE;
1706 switch (ctxt->modrm_reg) {
1708 emulate_1op(ctxt, "inc");
1711 emulate_1op(ctxt, "dec");
1713 case 2: /* call near abs */ {
1715 old_eip = ctxt->_eip;
1716 ctxt->_eip = ctxt->src.val;
1717 ctxt->src.val = old_eip;
1721 case 4: /* jmp abs */
1722 ctxt->_eip = ctxt->src.val;
1724 case 5: /* jmp far */
1725 rc = em_jmp_far(ctxt);
1734 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1736 u64 old = ctxt->dst.orig_val64;
1738 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
1739 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
1740 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1741 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1742 ctxt->eflags &= ~EFLG_ZF;
1744 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
1745 (u32) ctxt->regs[VCPU_REGS_RBX];
1747 ctxt->eflags |= EFLG_ZF;
1749 return X86EMUL_CONTINUE;
1752 static int em_ret(struct x86_emulate_ctxt *ctxt)
1754 ctxt->dst.type = OP_REG;
1755 ctxt->dst.addr.reg = &ctxt->_eip;
1756 ctxt->dst.bytes = ctxt->op_bytes;
1757 return em_pop(ctxt);
1760 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1765 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
1766 if (rc != X86EMUL_CONTINUE)
1768 if (ctxt->op_bytes == 4)
1769 ctxt->_eip = (u32)ctxt->_eip;
1770 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1771 if (rc != X86EMUL_CONTINUE)
1773 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1777 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1782 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1784 rc = load_segment_descriptor(ctxt, sel, seg);
1785 if (rc != X86EMUL_CONTINUE)
1788 ctxt->dst.val = ctxt->src.val;
1793 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1794 struct desc_struct *cs, struct desc_struct *ss)
1798 memset(cs, 0, sizeof(struct desc_struct));
1799 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1800 memset(ss, 0, sizeof(struct desc_struct));
1802 cs->l = 0; /* will be adjusted later */
1803 set_desc_base(cs, 0); /* flat segment */
1804 cs->g = 1; /* 4kb granularity */
1805 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1806 cs->type = 0x0b; /* Read, Execute, Accessed */
1808 cs->dpl = 0; /* will be adjusted later */
1812 set_desc_base(ss, 0); /* flat segment */
1813 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1814 ss->g = 1; /* 4kb granularity */
1816 ss->type = 0x03; /* Read/Write, Accessed */
1817 ss->d = 1; /* 32bit stack segment */
1822 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1824 struct x86_emulate_ops *ops = ctxt->ops;
1825 struct desc_struct cs, ss;
1830 /* syscall is not available in real mode */
1831 if (ctxt->mode == X86EMUL_MODE_REAL ||
1832 ctxt->mode == X86EMUL_MODE_VM86)
1833 return emulate_ud(ctxt);
1835 ops->get_msr(ctxt, MSR_EFER, &efer);
1836 setup_syscalls_segments(ctxt, &cs, &ss);
1838 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1840 cs_sel = (u16)(msr_data & 0xfffc);
1841 ss_sel = (u16)(msr_data + 8);
1843 if (efer & EFER_LMA) {
1847 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1848 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1850 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
1851 if (efer & EFER_LMA) {
1852 #ifdef CONFIG_X86_64
1853 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1856 ctxt->mode == X86EMUL_MODE_PROT64 ?
1857 MSR_LSTAR : MSR_CSTAR, &msr_data);
1858 ctxt->_eip = msr_data;
1860 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1861 ctxt->eflags &= ~(msr_data | EFLG_RF);
1865 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1866 ctxt->_eip = (u32)msr_data;
1868 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1871 return X86EMUL_CONTINUE;
1874 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1876 struct x86_emulate_ops *ops = ctxt->ops;
1877 struct desc_struct cs, ss;
1882 ops->get_msr(ctxt, MSR_EFER, &efer);
1883 /* inject #GP if in real mode */
1884 if (ctxt->mode == X86EMUL_MODE_REAL)
1885 return emulate_gp(ctxt, 0);
1887 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1888 * Therefore, we inject an #UD.
1890 if (ctxt->mode == X86EMUL_MODE_PROT64)
1891 return emulate_ud(ctxt);
1893 setup_syscalls_segments(ctxt, &cs, &ss);
1895 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1896 switch (ctxt->mode) {
1897 case X86EMUL_MODE_PROT32:
1898 if ((msr_data & 0xfffc) == 0x0)
1899 return emulate_gp(ctxt, 0);
1901 case X86EMUL_MODE_PROT64:
1902 if (msr_data == 0x0)
1903 return emulate_gp(ctxt, 0);
1907 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1908 cs_sel = (u16)msr_data;
1909 cs_sel &= ~SELECTOR_RPL_MASK;
1910 ss_sel = cs_sel + 8;
1911 ss_sel &= ~SELECTOR_RPL_MASK;
1912 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1917 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1918 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1920 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1921 ctxt->_eip = msr_data;
1923 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1924 ctxt->regs[VCPU_REGS_RSP] = msr_data;
1926 return X86EMUL_CONTINUE;
1929 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1931 struct x86_emulate_ops *ops = ctxt->ops;
1932 struct desc_struct cs, ss;
1935 u16 cs_sel = 0, ss_sel = 0;
1937 /* inject #GP if in real mode or Virtual 8086 mode */
1938 if (ctxt->mode == X86EMUL_MODE_REAL ||
1939 ctxt->mode == X86EMUL_MODE_VM86)
1940 return emulate_gp(ctxt, 0);
1942 setup_syscalls_segments(ctxt, &cs, &ss);
1944 if ((ctxt->rex_prefix & 0x8) != 0x0)
1945 usermode = X86EMUL_MODE_PROT64;
1947 usermode = X86EMUL_MODE_PROT32;
1951 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1953 case X86EMUL_MODE_PROT32:
1954 cs_sel = (u16)(msr_data + 16);
1955 if ((msr_data & 0xfffc) == 0x0)
1956 return emulate_gp(ctxt, 0);
1957 ss_sel = (u16)(msr_data + 24);
1959 case X86EMUL_MODE_PROT64:
1960 cs_sel = (u16)(msr_data + 32);
1961 if (msr_data == 0x0)
1962 return emulate_gp(ctxt, 0);
1963 ss_sel = cs_sel + 8;
1968 cs_sel |= SELECTOR_RPL_MASK;
1969 ss_sel |= SELECTOR_RPL_MASK;
1971 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1972 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1974 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
1975 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
1977 return X86EMUL_CONTINUE;
1980 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
1983 if (ctxt->mode == X86EMUL_MODE_REAL)
1985 if (ctxt->mode == X86EMUL_MODE_VM86)
1987 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1988 return ctxt->ops->cpl(ctxt) > iopl;
1991 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
1994 struct x86_emulate_ops *ops = ctxt->ops;
1995 struct desc_struct tr_seg;
1998 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
1999 unsigned mask = (1 << len) - 1;
2002 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2005 if (desc_limit_scaled(&tr_seg) < 103)
2007 base = get_desc_base(&tr_seg);
2008 #ifdef CONFIG_X86_64
2009 base |= ((u64)base3) << 32;
2011 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2012 if (r != X86EMUL_CONTINUE)
2014 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2016 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2017 if (r != X86EMUL_CONTINUE)
2019 if ((perm >> bit_idx) & mask)
2024 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2030 if (emulator_bad_iopl(ctxt))
2031 if (!emulator_io_port_access_allowed(ctxt, port, len))
2034 ctxt->perm_ok = true;
2039 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2040 struct tss_segment_16 *tss)
2042 tss->ip = ctxt->_eip;
2043 tss->flag = ctxt->eflags;
2044 tss->ax = ctxt->regs[VCPU_REGS_RAX];
2045 tss->cx = ctxt->regs[VCPU_REGS_RCX];
2046 tss->dx = ctxt->regs[VCPU_REGS_RDX];
2047 tss->bx = ctxt->regs[VCPU_REGS_RBX];
2048 tss->sp = ctxt->regs[VCPU_REGS_RSP];
2049 tss->bp = ctxt->regs[VCPU_REGS_RBP];
2050 tss->si = ctxt->regs[VCPU_REGS_RSI];
2051 tss->di = ctxt->regs[VCPU_REGS_RDI];
2053 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2054 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2055 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2056 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2057 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2060 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2061 struct tss_segment_16 *tss)
2065 ctxt->_eip = tss->ip;
2066 ctxt->eflags = tss->flag | 2;
2067 ctxt->regs[VCPU_REGS_RAX] = tss->ax;
2068 ctxt->regs[VCPU_REGS_RCX] = tss->cx;
2069 ctxt->regs[VCPU_REGS_RDX] = tss->dx;
2070 ctxt->regs[VCPU_REGS_RBX] = tss->bx;
2071 ctxt->regs[VCPU_REGS_RSP] = tss->sp;
2072 ctxt->regs[VCPU_REGS_RBP] = tss->bp;
2073 ctxt->regs[VCPU_REGS_RSI] = tss->si;
2074 ctxt->regs[VCPU_REGS_RDI] = tss->di;
2077 * SDM says that segment selectors are loaded before segment
2080 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2081 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2082 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2083 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2084 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2087 * Now load segment descriptors. If fault happenes at this stage
2088 * it is handled in a context of new task
2090 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2091 if (ret != X86EMUL_CONTINUE)
2093 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2094 if (ret != X86EMUL_CONTINUE)
2096 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2097 if (ret != X86EMUL_CONTINUE)
2099 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2100 if (ret != X86EMUL_CONTINUE)
2102 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2103 if (ret != X86EMUL_CONTINUE)
2106 return X86EMUL_CONTINUE;
2109 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2110 u16 tss_selector, u16 old_tss_sel,
2111 ulong old_tss_base, struct desc_struct *new_desc)
2113 struct x86_emulate_ops *ops = ctxt->ops;
2114 struct tss_segment_16 tss_seg;
2116 u32 new_tss_base = get_desc_base(new_desc);
2118 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2120 if (ret != X86EMUL_CONTINUE)
2121 /* FIXME: need to provide precise fault address */
2124 save_state_to_tss16(ctxt, &tss_seg);
2126 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2128 if (ret != X86EMUL_CONTINUE)
2129 /* FIXME: need to provide precise fault address */
2132 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2134 if (ret != X86EMUL_CONTINUE)
2135 /* FIXME: need to provide precise fault address */
2138 if (old_tss_sel != 0xffff) {
2139 tss_seg.prev_task_link = old_tss_sel;
2141 ret = ops->write_std(ctxt, new_tss_base,
2142 &tss_seg.prev_task_link,
2143 sizeof tss_seg.prev_task_link,
2145 if (ret != X86EMUL_CONTINUE)
2146 /* FIXME: need to provide precise fault address */
2150 return load_state_from_tss16(ctxt, &tss_seg);
2153 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2154 struct tss_segment_32 *tss)
2156 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2157 tss->eip = ctxt->_eip;
2158 tss->eflags = ctxt->eflags;
2159 tss->eax = ctxt->regs[VCPU_REGS_RAX];
2160 tss->ecx = ctxt->regs[VCPU_REGS_RCX];
2161 tss->edx = ctxt->regs[VCPU_REGS_RDX];
2162 tss->ebx = ctxt->regs[VCPU_REGS_RBX];
2163 tss->esp = ctxt->regs[VCPU_REGS_RSP];
2164 tss->ebp = ctxt->regs[VCPU_REGS_RBP];
2165 tss->esi = ctxt->regs[VCPU_REGS_RSI];
2166 tss->edi = ctxt->regs[VCPU_REGS_RDI];
2168 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2169 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2170 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2171 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2172 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2173 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2174 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2177 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2178 struct tss_segment_32 *tss)
2182 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2183 return emulate_gp(ctxt, 0);
2184 ctxt->_eip = tss->eip;
2185 ctxt->eflags = tss->eflags | 2;
2186 ctxt->regs[VCPU_REGS_RAX] = tss->eax;
2187 ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
2188 ctxt->regs[VCPU_REGS_RDX] = tss->edx;
2189 ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
2190 ctxt->regs[VCPU_REGS_RSP] = tss->esp;
2191 ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
2192 ctxt->regs[VCPU_REGS_RSI] = tss->esi;
2193 ctxt->regs[VCPU_REGS_RDI] = tss->edi;
2196 * SDM says that segment selectors are loaded before segment
2199 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2200 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2201 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2202 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2203 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2204 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2205 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2208 * Now load segment descriptors. If fault happenes at this stage
2209 * it is handled in a context of new task
2211 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2212 if (ret != X86EMUL_CONTINUE)
2214 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2215 if (ret != X86EMUL_CONTINUE)
2217 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2218 if (ret != X86EMUL_CONTINUE)
2220 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2221 if (ret != X86EMUL_CONTINUE)
2223 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2224 if (ret != X86EMUL_CONTINUE)
2226 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2227 if (ret != X86EMUL_CONTINUE)
2229 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2230 if (ret != X86EMUL_CONTINUE)
2233 return X86EMUL_CONTINUE;
2236 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2237 u16 tss_selector, u16 old_tss_sel,
2238 ulong old_tss_base, struct desc_struct *new_desc)
2240 struct x86_emulate_ops *ops = ctxt->ops;
2241 struct tss_segment_32 tss_seg;
2243 u32 new_tss_base = get_desc_base(new_desc);
2245 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2247 if (ret != X86EMUL_CONTINUE)
2248 /* FIXME: need to provide precise fault address */
2251 save_state_to_tss32(ctxt, &tss_seg);
2253 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2255 if (ret != X86EMUL_CONTINUE)
2256 /* FIXME: need to provide precise fault address */
2259 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2261 if (ret != X86EMUL_CONTINUE)
2262 /* FIXME: need to provide precise fault address */
2265 if (old_tss_sel != 0xffff) {
2266 tss_seg.prev_task_link = old_tss_sel;
2268 ret = ops->write_std(ctxt, new_tss_base,
2269 &tss_seg.prev_task_link,
2270 sizeof tss_seg.prev_task_link,
2272 if (ret != X86EMUL_CONTINUE)
2273 /* FIXME: need to provide precise fault address */
2277 return load_state_from_tss32(ctxt, &tss_seg);
2280 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2281 u16 tss_selector, int reason,
2282 bool has_error_code, u32 error_code)
2284 struct x86_emulate_ops *ops = ctxt->ops;
2285 struct desc_struct curr_tss_desc, next_tss_desc;
2287 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2288 ulong old_tss_base =
2289 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2292 /* FIXME: old_tss_base == ~0 ? */
2294 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2295 if (ret != X86EMUL_CONTINUE)
2297 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2298 if (ret != X86EMUL_CONTINUE)
2301 /* FIXME: check that next_tss_desc is tss */
2303 if (reason != TASK_SWITCH_IRET) {
2304 if ((tss_selector & 3) > next_tss_desc.dpl ||
2305 ops->cpl(ctxt) > next_tss_desc.dpl)
2306 return emulate_gp(ctxt, 0);
2309 desc_limit = desc_limit_scaled(&next_tss_desc);
2310 if (!next_tss_desc.p ||
2311 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2312 desc_limit < 0x2b)) {
2313 emulate_ts(ctxt, tss_selector & 0xfffc);
2314 return X86EMUL_PROPAGATE_FAULT;
2317 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2318 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2319 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2322 if (reason == TASK_SWITCH_IRET)
2323 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2325 /* set back link to prev task only if NT bit is set in eflags
2326 note that old_tss_sel is not used afetr this point */
2327 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2328 old_tss_sel = 0xffff;
2330 if (next_tss_desc.type & 8)
2331 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2332 old_tss_base, &next_tss_desc);
2334 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2335 old_tss_base, &next_tss_desc);
2336 if (ret != X86EMUL_CONTINUE)
2339 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2340 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2342 if (reason != TASK_SWITCH_IRET) {
2343 next_tss_desc.type |= (1 << 1); /* set busy flag */
2344 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2347 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2348 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2350 if (has_error_code) {
2351 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2352 ctxt->lock_prefix = 0;
2353 ctxt->src.val = (unsigned long) error_code;
2354 ret = em_push(ctxt);
2360 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2361 u16 tss_selector, int reason,
2362 bool has_error_code, u32 error_code)
2366 ctxt->_eip = ctxt->eip;
2367 ctxt->dst.type = OP_NONE;
2369 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2370 has_error_code, error_code);
2372 if (rc == X86EMUL_CONTINUE)
2373 ctxt->eip = ctxt->_eip;
2375 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2378 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2379 int reg, struct operand *op)
2381 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2383 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
2384 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
2385 op->addr.mem.seg = seg;
2388 static int em_das(struct x86_emulate_ctxt *ctxt)
2391 bool af, cf, old_cf;
2393 cf = ctxt->eflags & X86_EFLAGS_CF;
2399 af = ctxt->eflags & X86_EFLAGS_AF;
2400 if ((al & 0x0f) > 9 || af) {
2402 cf = old_cf | (al >= 250);
2407 if (old_al > 0x99 || old_cf) {
2413 /* Set PF, ZF, SF */
2414 ctxt->src.type = OP_IMM;
2416 ctxt->src.bytes = 1;
2417 emulate_2op_SrcV(ctxt, "or");
2418 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2420 ctxt->eflags |= X86_EFLAGS_CF;
2422 ctxt->eflags |= X86_EFLAGS_AF;
2423 return X86EMUL_CONTINUE;
2426 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2432 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2433 old_eip = ctxt->_eip;
2435 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2436 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2437 return X86EMUL_CONTINUE;
2440 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2442 ctxt->src.val = old_cs;
2444 if (rc != X86EMUL_CONTINUE)
2447 ctxt->src.val = old_eip;
2448 return em_push(ctxt);
2451 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2455 ctxt->dst.type = OP_REG;
2456 ctxt->dst.addr.reg = &ctxt->_eip;
2457 ctxt->dst.bytes = ctxt->op_bytes;
2458 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2459 if (rc != X86EMUL_CONTINUE)
2461 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RSP], ctxt->src.val);
2462 return X86EMUL_CONTINUE;
2465 static int em_add(struct x86_emulate_ctxt *ctxt)
2467 emulate_2op_SrcV(ctxt, "add");
2468 return X86EMUL_CONTINUE;
2471 static int em_or(struct x86_emulate_ctxt *ctxt)
2473 emulate_2op_SrcV(ctxt, "or");
2474 return X86EMUL_CONTINUE;
2477 static int em_adc(struct x86_emulate_ctxt *ctxt)
2479 emulate_2op_SrcV(ctxt, "adc");
2480 return X86EMUL_CONTINUE;
2483 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2485 emulate_2op_SrcV(ctxt, "sbb");
2486 return X86EMUL_CONTINUE;
2489 static int em_and(struct x86_emulate_ctxt *ctxt)
2491 emulate_2op_SrcV(ctxt, "and");
2492 return X86EMUL_CONTINUE;
2495 static int em_sub(struct x86_emulate_ctxt *ctxt)
2497 emulate_2op_SrcV(ctxt, "sub");
2498 return X86EMUL_CONTINUE;
2501 static int em_xor(struct x86_emulate_ctxt *ctxt)
2503 emulate_2op_SrcV(ctxt, "xor");
2504 return X86EMUL_CONTINUE;
2507 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2509 emulate_2op_SrcV(ctxt, "cmp");
2510 /* Disable writeback. */
2511 ctxt->dst.type = OP_NONE;
2512 return X86EMUL_CONTINUE;
2515 static int em_test(struct x86_emulate_ctxt *ctxt)
2517 emulate_2op_SrcV(ctxt, "test");
2518 /* Disable writeback. */
2519 ctxt->dst.type = OP_NONE;
2520 return X86EMUL_CONTINUE;
2523 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2525 /* Write back the register source. */
2526 ctxt->src.val = ctxt->dst.val;
2527 write_register_operand(&ctxt->src);
2529 /* Write back the memory destination with implicit LOCK prefix. */
2530 ctxt->dst.val = ctxt->src.orig_val;
2531 ctxt->lock_prefix = 1;
2532 return X86EMUL_CONTINUE;
2535 static int em_imul(struct x86_emulate_ctxt *ctxt)
2537 emulate_2op_SrcV_nobyte(ctxt, "imul");
2538 return X86EMUL_CONTINUE;
2541 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2543 ctxt->dst.val = ctxt->src2.val;
2544 return em_imul(ctxt);
2547 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2549 ctxt->dst.type = OP_REG;
2550 ctxt->dst.bytes = ctxt->src.bytes;
2551 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
2552 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2554 return X86EMUL_CONTINUE;
2557 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2561 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2562 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
2563 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
2564 return X86EMUL_CONTINUE;
2567 static int em_mov(struct x86_emulate_ctxt *ctxt)
2569 ctxt->dst.val = ctxt->src.val;
2570 return X86EMUL_CONTINUE;
2573 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
2575 if (ctxt->modrm_reg > VCPU_SREG_GS)
2576 return emulate_ud(ctxt);
2578 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
2579 return X86EMUL_CONTINUE;
2582 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
2584 u16 sel = ctxt->src.val;
2586 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
2587 return emulate_ud(ctxt);
2589 if (ctxt->modrm_reg == VCPU_SREG_SS)
2590 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2592 /* Disable writeback. */
2593 ctxt->dst.type = OP_NONE;
2594 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
2597 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2599 memcpy(&ctxt->dst.vec_val, &ctxt->src.vec_val, ctxt->op_bytes);
2600 return X86EMUL_CONTINUE;
2603 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2608 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
2609 if (rc == X86EMUL_CONTINUE)
2610 ctxt->ops->invlpg(ctxt, linear);
2611 /* Disable writeback. */
2612 ctxt->dst.type = OP_NONE;
2613 return X86EMUL_CONTINUE;
2616 static int em_clts(struct x86_emulate_ctxt *ctxt)
2620 cr0 = ctxt->ops->get_cr(ctxt, 0);
2622 ctxt->ops->set_cr(ctxt, 0, cr0);
2623 return X86EMUL_CONTINUE;
2626 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2630 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
2631 return X86EMUL_UNHANDLEABLE;
2633 rc = ctxt->ops->fix_hypercall(ctxt);
2634 if (rc != X86EMUL_CONTINUE)
2637 /* Let the processor re-execute the fixed hypercall */
2638 ctxt->_eip = ctxt->eip;
2639 /* Disable writeback. */
2640 ctxt->dst.type = OP_NONE;
2641 return X86EMUL_CONTINUE;
2644 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2646 struct desc_ptr desc_ptr;
2649 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2650 &desc_ptr.size, &desc_ptr.address,
2652 if (rc != X86EMUL_CONTINUE)
2654 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2655 /* Disable writeback. */
2656 ctxt->dst.type = OP_NONE;
2657 return X86EMUL_CONTINUE;
2660 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2664 rc = ctxt->ops->fix_hypercall(ctxt);
2666 /* Disable writeback. */
2667 ctxt->dst.type = OP_NONE;
2671 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2673 struct desc_ptr desc_ptr;
2676 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
2677 &desc_ptr.size, &desc_ptr.address,
2679 if (rc != X86EMUL_CONTINUE)
2681 ctxt->ops->set_idt(ctxt, &desc_ptr);
2682 /* Disable writeback. */
2683 ctxt->dst.type = OP_NONE;
2684 return X86EMUL_CONTINUE;
2687 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2689 ctxt->dst.bytes = 2;
2690 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
2691 return X86EMUL_CONTINUE;
2694 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2696 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2697 | (ctxt->src.val & 0x0f));
2698 ctxt->dst.type = OP_NONE;
2699 return X86EMUL_CONTINUE;
2702 static int em_loop(struct x86_emulate_ctxt *ctxt)
2704 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
2705 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
2706 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
2707 jmp_rel(ctxt, ctxt->src.val);
2709 return X86EMUL_CONTINUE;
2712 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
2714 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
2715 jmp_rel(ctxt, ctxt->src.val);
2717 return X86EMUL_CONTINUE;
2720 static int em_cli(struct x86_emulate_ctxt *ctxt)
2722 if (emulator_bad_iopl(ctxt))
2723 return emulate_gp(ctxt, 0);
2725 ctxt->eflags &= ~X86_EFLAGS_IF;
2726 return X86EMUL_CONTINUE;
2729 static int em_sti(struct x86_emulate_ctxt *ctxt)
2731 if (emulator_bad_iopl(ctxt))
2732 return emulate_gp(ctxt, 0);
2734 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
2735 ctxt->eflags |= X86_EFLAGS_IF;
2736 return X86EMUL_CONTINUE;
2739 static bool valid_cr(int nr)
2751 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2753 if (!valid_cr(ctxt->modrm_reg))
2754 return emulate_ud(ctxt);
2756 return X86EMUL_CONTINUE;
2759 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2761 u64 new_val = ctxt->src.val64;
2762 int cr = ctxt->modrm_reg;
2765 static u64 cr_reserved_bits[] = {
2766 0xffffffff00000000ULL,
2767 0, 0, 0, /* CR3 checked later */
2774 return emulate_ud(ctxt);
2776 if (new_val & cr_reserved_bits[cr])
2777 return emulate_gp(ctxt, 0);
2782 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2783 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2784 return emulate_gp(ctxt, 0);
2786 cr4 = ctxt->ops->get_cr(ctxt, 4);
2787 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2789 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2790 !(cr4 & X86_CR4_PAE))
2791 return emulate_gp(ctxt, 0);
2798 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2799 if (efer & EFER_LMA)
2800 rsvd = CR3_L_MODE_RESERVED_BITS;
2801 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2802 rsvd = CR3_PAE_RESERVED_BITS;
2803 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2804 rsvd = CR3_NONPAE_RESERVED_BITS;
2807 return emulate_gp(ctxt, 0);
2814 cr4 = ctxt->ops->get_cr(ctxt, 4);
2815 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2817 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2818 return emulate_gp(ctxt, 0);
2824 return X86EMUL_CONTINUE;
2827 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2831 ctxt->ops->get_dr(ctxt, 7, &dr7);
2833 /* Check if DR7.Global_Enable is set */
2834 return dr7 & (1 << 13);
2837 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2839 int dr = ctxt->modrm_reg;
2843 return emulate_ud(ctxt);
2845 cr4 = ctxt->ops->get_cr(ctxt, 4);
2846 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2847 return emulate_ud(ctxt);
2849 if (check_dr7_gd(ctxt))
2850 return emulate_db(ctxt);
2852 return X86EMUL_CONTINUE;
2855 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2857 u64 new_val = ctxt->src.val64;
2858 int dr = ctxt->modrm_reg;
2860 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2861 return emulate_gp(ctxt, 0);
2863 return check_dr_read(ctxt);
2866 static int check_svme(struct x86_emulate_ctxt *ctxt)
2870 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2872 if (!(efer & EFER_SVME))
2873 return emulate_ud(ctxt);
2875 return X86EMUL_CONTINUE;
2878 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2880 u64 rax = ctxt->regs[VCPU_REGS_RAX];
2882 /* Valid physical address? */
2883 if (rax & 0xffff000000000000ULL)
2884 return emulate_gp(ctxt, 0);
2886 return check_svme(ctxt);
2889 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2891 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2893 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2894 return emulate_ud(ctxt);
2896 return X86EMUL_CONTINUE;
2899 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2901 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2902 u64 rcx = ctxt->regs[VCPU_REGS_RCX];
2904 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2906 return emulate_gp(ctxt, 0);
2908 return X86EMUL_CONTINUE;
2911 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2913 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
2914 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
2915 return emulate_gp(ctxt, 0);
2917 return X86EMUL_CONTINUE;
2920 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2922 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
2923 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
2924 return emulate_gp(ctxt, 0);
2926 return X86EMUL_CONTINUE;
2929 #define D(_y) { .flags = (_y) }
2930 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2931 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2932 .check_perm = (_p) }
2934 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2935 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2936 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2937 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2938 #define II(_f, _e, _i) \
2939 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
2940 #define IIP(_f, _e, _i, _p) \
2941 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
2942 .check_perm = (_p) }
2943 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
2945 #define D2bv(_f) D((_f) | ByteOp), D(_f)
2946 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
2947 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
2949 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
2950 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
2951 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
2953 static struct opcode group7_rm1[] = {
2954 DI(SrcNone | ModRM | Priv, monitor),
2955 DI(SrcNone | ModRM | Priv, mwait),
2959 static struct opcode group7_rm3[] = {
2960 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
2961 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
2962 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
2963 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
2964 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
2965 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
2966 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
2967 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
2970 static struct opcode group7_rm7[] = {
2972 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
2976 static struct opcode group1[] = {
2987 static struct opcode group1A[] = {
2988 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
2991 static struct opcode group3[] = {
2992 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
2993 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
2994 X4(D(SrcMem | ModRM)),
2997 static struct opcode group4[] = {
2998 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3002 static struct opcode group5[] = {
3003 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3004 D(SrcMem | ModRM | Stack),
3005 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3006 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3007 D(SrcMem | ModRM | Stack), N,
3010 static struct opcode group6[] = {
3011 DI(ModRM | Prot, sldt),
3012 DI(ModRM | Prot, str),
3013 DI(ModRM | Prot | Priv, lldt),
3014 DI(ModRM | Prot | Priv, ltr),
3018 static struct group_dual group7 = { {
3019 DI(ModRM | Mov | DstMem | Priv, sgdt),
3020 DI(ModRM | Mov | DstMem | Priv, sidt),
3021 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3022 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3023 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3024 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3025 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3027 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3029 N, EXT(0, group7_rm3),
3030 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3031 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3034 static struct opcode group8[] = {
3036 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3037 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3040 static struct group_dual group9 = { {
3041 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3043 N, N, N, N, N, N, N, N,
3046 static struct opcode group11[] = {
3047 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3050 static struct gprefix pfx_0f_6f_0f_7f = {
3051 N, N, N, I(Sse, em_movdqu),
3054 static struct opcode opcode_table[256] = {
3056 I6ALU(Lock, em_add),
3057 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3060 D(ImplicitOps | Stack | No64), N,
3062 I6ALU(Lock, em_adc),
3063 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3065 I6ALU(Lock, em_sbb),
3066 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3068 I6ALU(Lock, em_and), N, N,
3070 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3072 I6ALU(Lock, em_xor), N, N,
3074 I6ALU(0, em_cmp), N, N,
3078 X8(I(SrcReg | Stack, em_push)),
3080 X8(I(DstReg | Stack, em_pop)),
3082 I(ImplicitOps | Stack | No64, em_pusha),
3083 I(ImplicitOps | Stack | No64, em_popa),
3084 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3087 I(SrcImm | Mov | Stack, em_push),
3088 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3089 I(SrcImmByte | Mov | Stack, em_push),
3090 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3091 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3092 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3096 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3097 G(DstMem | SrcImm | ModRM | Group, group1),
3098 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3099 G(DstMem | SrcImmByte | ModRM | Group, group1),
3100 I2bv(DstMem | SrcReg | ModRM, em_test),
3101 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3103 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3104 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3105 I(DstMem | SrcNone | ModRM | Mov, em_mov_rm_sreg),
3106 D(ModRM | SrcMem | NoAccess | DstReg),
3107 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3110 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3112 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3113 I(SrcImmFAddr | No64, em_call_far), N,
3114 II(ImplicitOps | Stack, em_pushf, pushf),
3115 II(ImplicitOps | Stack, em_popf, popf), N, N,
3117 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3118 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3119 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3120 I2bv(SrcSI | DstDI | String, em_cmp),
3122 I2bv(DstAcc | SrcImm, em_test),
3123 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3124 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3125 I2bv(SrcAcc | DstDI | String, em_cmp),
3127 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3129 X8(I(DstReg | SrcImm | Mov, em_mov)),
3131 D2bv(DstMem | SrcImmByte | ModRM),
3132 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3133 I(ImplicitOps | Stack, em_ret),
3134 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3135 G(ByteOp, group11), G(0, group11),
3137 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3138 D(ImplicitOps), DI(SrcImmByte, intn),
3139 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3141 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3144 N, N, N, N, N, N, N, N,
3146 X3(I(SrcImmByte, em_loop)),
3147 I(SrcImmByte, em_jcxz),
3148 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3149 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3151 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3152 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3153 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3154 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3156 N, DI(ImplicitOps, icebp), N, N,
3157 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3158 G(ByteOp, group3), G(0, group3),
3160 D(ImplicitOps), D(ImplicitOps),
3161 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3162 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3165 static struct opcode twobyte_table[256] = {
3167 G(0, group6), GD(0, &group7), N, N,
3168 N, I(ImplicitOps | VendorSpecific, em_syscall),
3169 II(ImplicitOps | Priv, em_clts, clts), N,
3170 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3171 N, D(ImplicitOps | ModRM), N, N,
3173 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3175 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3176 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3177 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3178 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3180 N, N, N, N, N, N, N, N,
3182 DI(ImplicitOps | Priv, wrmsr),
3183 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3184 DI(ImplicitOps | Priv, rdmsr),
3185 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3186 I(ImplicitOps | VendorSpecific, em_sysenter),
3187 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3189 N, N, N, N, N, N, N, N,
3191 X16(D(DstReg | SrcMem | ModRM | Mov)),
3193 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3198 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3203 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3207 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3209 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3210 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3211 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3212 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3214 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3215 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3216 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3217 D(DstMem | SrcReg | Src2CL | ModRM),
3218 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3220 D2bv(DstMem | SrcReg | ModRM | Lock),
3221 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3222 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3223 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3226 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3227 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3228 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3230 D2bv(DstMem | SrcReg | ModRM | Lock),
3231 N, D(DstMem | SrcReg | ModRM | Mov),
3232 N, N, N, GD(0, &group9),
3233 N, N, N, N, N, N, N, N,
3235 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3237 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3239 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3255 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3259 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3265 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3266 unsigned size, bool sign_extension)
3268 int rc = X86EMUL_CONTINUE;
3272 op->addr.mem.ea = ctxt->_eip;
3273 /* NB. Immediates are sign-extended as necessary. */
3274 switch (op->bytes) {
3276 op->val = insn_fetch(s8, ctxt);
3279 op->val = insn_fetch(s16, ctxt);
3282 op->val = insn_fetch(s32, ctxt);
3285 if (!sign_extension) {
3286 switch (op->bytes) {
3294 op->val &= 0xffffffff;
3302 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3304 int rc = X86EMUL_CONTINUE;
3305 int mode = ctxt->mode;
3306 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3307 bool op_prefix = false;
3308 struct opcode opcode;
3309 struct operand memop = { .type = OP_NONE }, *memopp = NULL;
3311 ctxt->_eip = ctxt->eip;
3312 ctxt->fetch.start = ctxt->_eip;
3313 ctxt->fetch.end = ctxt->fetch.start + insn_len;
3315 memcpy(ctxt->fetch.data, insn, insn_len);
3318 case X86EMUL_MODE_REAL:
3319 case X86EMUL_MODE_VM86:
3320 case X86EMUL_MODE_PROT16:
3321 def_op_bytes = def_ad_bytes = 2;
3323 case X86EMUL_MODE_PROT32:
3324 def_op_bytes = def_ad_bytes = 4;
3326 #ifdef CONFIG_X86_64
3327 case X86EMUL_MODE_PROT64:
3333 return EMULATION_FAILED;
3336 ctxt->op_bytes = def_op_bytes;
3337 ctxt->ad_bytes = def_ad_bytes;
3339 /* Legacy prefixes. */
3341 switch (ctxt->b = insn_fetch(u8, ctxt)) {
3342 case 0x66: /* operand-size override */
3344 /* switch between 2/4 bytes */
3345 ctxt->op_bytes = def_op_bytes ^ 6;
3347 case 0x67: /* address-size override */
3348 if (mode == X86EMUL_MODE_PROT64)
3349 /* switch between 4/8 bytes */
3350 ctxt->ad_bytes = def_ad_bytes ^ 12;
3352 /* switch between 2/4 bytes */
3353 ctxt->ad_bytes = def_ad_bytes ^ 6;
3355 case 0x26: /* ES override */
3356 case 0x2e: /* CS override */
3357 case 0x36: /* SS override */
3358 case 0x3e: /* DS override */
3359 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
3361 case 0x64: /* FS override */
3362 case 0x65: /* GS override */
3363 set_seg_override(ctxt, ctxt->b & 7);
3365 case 0x40 ... 0x4f: /* REX */
3366 if (mode != X86EMUL_MODE_PROT64)
3368 ctxt->rex_prefix = ctxt->b;
3370 case 0xf0: /* LOCK */
3371 ctxt->lock_prefix = 1;
3373 case 0xf2: /* REPNE/REPNZ */
3374 case 0xf3: /* REP/REPE/REPZ */
3375 ctxt->rep_prefix = ctxt->b;
3381 /* Any legacy prefix after a REX prefix nullifies its effect. */
3383 ctxt->rex_prefix = 0;
3389 if (ctxt->rex_prefix & 8)
3390 ctxt->op_bytes = 8; /* REX.W */
3392 /* Opcode byte(s). */
3393 opcode = opcode_table[ctxt->b];
3394 /* Two-byte opcode? */
3395 if (ctxt->b == 0x0f) {
3397 ctxt->b = insn_fetch(u8, ctxt);
3398 opcode = twobyte_table[ctxt->b];
3400 ctxt->d = opcode.flags;
3402 while (ctxt->d & GroupMask) {
3403 switch (ctxt->d & GroupMask) {
3405 ctxt->modrm = insn_fetch(u8, ctxt);
3407 goffset = (ctxt->modrm >> 3) & 7;
3408 opcode = opcode.u.group[goffset];
3411 ctxt->modrm = insn_fetch(u8, ctxt);
3413 goffset = (ctxt->modrm >> 3) & 7;
3414 if ((ctxt->modrm >> 6) == 3)
3415 opcode = opcode.u.gdual->mod3[goffset];
3417 opcode = opcode.u.gdual->mod012[goffset];
3420 goffset = ctxt->modrm & 7;
3421 opcode = opcode.u.group[goffset];
3424 if (ctxt->rep_prefix && op_prefix)
3425 return EMULATION_FAILED;
3426 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
3427 switch (simd_prefix) {
3428 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3429 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3430 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3431 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3435 return EMULATION_FAILED;
3438 ctxt->d &= ~GroupMask;
3439 ctxt->d |= opcode.flags;
3442 ctxt->execute = opcode.u.execute;
3443 ctxt->check_perm = opcode.check_perm;
3444 ctxt->intercept = opcode.intercept;
3447 if (ctxt->d == 0 || (ctxt->d & Undefined))
3448 return EMULATION_FAILED;
3450 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3451 return EMULATION_FAILED;
3453 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
3456 if (ctxt->d & Op3264) {
3457 if (mode == X86EMUL_MODE_PROT64)
3464 ctxt->op_bytes = 16;
3466 /* ModRM and SIB bytes. */
3467 if (ctxt->d & ModRM) {
3468 rc = decode_modrm(ctxt, &memop);
3469 if (!ctxt->has_seg_override)
3470 set_seg_override(ctxt, ctxt->modrm_seg);
3471 } else if (ctxt->d & MemAbs)
3472 rc = decode_abs(ctxt, &memop);
3473 if (rc != X86EMUL_CONTINUE)
3476 if (!ctxt->has_seg_override)
3477 set_seg_override(ctxt, VCPU_SREG_DS);
3479 memop.addr.mem.seg = seg_override(ctxt);
3481 if (memop.type == OP_MEM && ctxt->ad_bytes != 8)
3482 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3485 * Decode and fetch the source operand: register, memory
3488 switch (ctxt->d & SrcMask) {
3492 decode_register_operand(ctxt, &ctxt->src, 0);
3501 memop.bytes = (ctxt->d & ByteOp) ? 1 :
3505 memopp = &ctxt->src;
3508 rc = decode_imm(ctxt, &ctxt->src, 2, false);
3511 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), true);
3514 rc = decode_imm(ctxt, &ctxt->src, imm_size(ctxt), false);
3517 rc = decode_imm(ctxt, &ctxt->src, 1, true);
3520 rc = decode_imm(ctxt, &ctxt->src, 1, false);
3523 ctxt->src.type = OP_REG;
3524 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3525 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3526 fetch_register_operand(&ctxt->src);
3529 ctxt->src.bytes = 1;
3533 ctxt->src.type = OP_MEM;
3534 ctxt->src.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3535 ctxt->src.addr.mem.ea =
3536 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
3537 ctxt->src.addr.mem.seg = seg_override(ctxt);
3541 ctxt->src.type = OP_IMM;
3542 ctxt->src.addr.mem.ea = ctxt->_eip;
3543 ctxt->src.bytes = ctxt->op_bytes + 2;
3544 insn_fetch_arr(ctxt->src.valptr, ctxt->src.bytes, ctxt);
3547 memop.bytes = ctxt->op_bytes + 2;
3551 ctxt->src.type = OP_REG;
3552 ctxt->src.bytes = 2;
3553 ctxt->src.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3554 fetch_register_operand(&ctxt->src);
3558 if (rc != X86EMUL_CONTINUE)
3562 * Decode and fetch the second source operand: register, memory
3565 switch (ctxt->d & Src2Mask) {
3569 ctxt->src2.bytes = 1;
3570 ctxt->src2.val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
3573 rc = decode_imm(ctxt, &ctxt->src2, 1, true);
3576 ctxt->src2.bytes = 1;
3580 rc = decode_imm(ctxt, &ctxt->src2, imm_size(ctxt), true);
3584 if (rc != X86EMUL_CONTINUE)
3587 /* Decode and fetch the destination operand: register or memory. */
3588 switch (ctxt->d & DstMask) {
3590 decode_register_operand(ctxt, &ctxt->dst,
3591 ctxt->twobyte && (ctxt->b == 0xb6 || ctxt->b == 0xb7));
3594 ctxt->dst.type = OP_IMM;
3595 ctxt->dst.addr.mem.ea = ctxt->_eip;
3596 ctxt->dst.bytes = 1;
3597 ctxt->dst.val = insn_fetch(u8, ctxt);
3602 memopp = &ctxt->dst;
3603 if ((ctxt->d & DstMask) == DstMem64)
3604 ctxt->dst.bytes = 8;
3606 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3607 if (ctxt->d & BitOp)
3608 fetch_bit_operand(ctxt);
3609 ctxt->dst.orig_val = ctxt->dst.val;
3612 ctxt->dst.type = OP_REG;
3613 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3614 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RAX];
3615 fetch_register_operand(&ctxt->dst);
3616 ctxt->dst.orig_val = ctxt->dst.val;
3619 ctxt->dst.type = OP_MEM;
3620 ctxt->dst.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3621 ctxt->dst.addr.mem.ea =
3622 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
3623 ctxt->dst.addr.mem.seg = VCPU_SREG_ES;
3627 ctxt->dst.type = OP_REG;
3628 ctxt->dst.bytes = 2;
3629 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
3630 fetch_register_operand(&ctxt->dst);
3633 /* Special instructions do their own operand decoding. */
3635 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3640 if (memopp && memopp->type == OP_MEM && ctxt->rip_relative)
3641 memopp->addr.mem.ea += ctxt->_eip;
3643 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
3646 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3648 /* The second termination condition only applies for REPE
3649 * and REPNE. Test if the repeat string operation prefix is
3650 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3651 * corresponding termination condition according to:
3652 * - if REPE/REPZ and ZF = 0 then done
3653 * - if REPNE/REPNZ and ZF = 1 then done
3655 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
3656 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
3657 && (((ctxt->rep_prefix == REPE_PREFIX) &&
3658 ((ctxt->eflags & EFLG_ZF) == 0))
3659 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
3660 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3666 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3668 struct x86_emulate_ops *ops = ctxt->ops;
3670 int rc = X86EMUL_CONTINUE;
3671 int saved_dst_type = ctxt->dst.type;
3673 ctxt->mem_read.pos = 0;
3675 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
3676 rc = emulate_ud(ctxt);
3680 /* LOCK prefix is allowed only with some instructions */
3681 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
3682 rc = emulate_ud(ctxt);
3686 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
3687 rc = emulate_ud(ctxt);
3692 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3693 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3694 rc = emulate_ud(ctxt);
3698 if ((ctxt->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3699 rc = emulate_nm(ctxt);
3703 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3704 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3705 X86_ICPT_PRE_EXCEPT);
3706 if (rc != X86EMUL_CONTINUE)
3710 /* Privileged instruction can be executed only in CPL=0 */
3711 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
3712 rc = emulate_gp(ctxt, 0);
3716 /* Instruction can only be executed in protected mode */
3717 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3718 rc = emulate_ud(ctxt);
3722 /* Do instruction specific permission checks */
3723 if (ctxt->check_perm) {
3724 rc = ctxt->check_perm(ctxt);
3725 if (rc != X86EMUL_CONTINUE)
3729 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3730 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3731 X86_ICPT_POST_EXCEPT);
3732 if (rc != X86EMUL_CONTINUE)
3736 if (ctxt->rep_prefix && (ctxt->d & String)) {
3737 /* All REP prefixes have the same first termination condition */
3738 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
3739 ctxt->eip = ctxt->_eip;
3744 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
3745 rc = segmented_read(ctxt, ctxt->src.addr.mem,
3746 ctxt->src.valptr, ctxt->src.bytes);
3747 if (rc != X86EMUL_CONTINUE)
3749 ctxt->src.orig_val64 = ctxt->src.val64;
3752 if (ctxt->src2.type == OP_MEM) {
3753 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
3754 &ctxt->src2.val, ctxt->src2.bytes);
3755 if (rc != X86EMUL_CONTINUE)
3759 if ((ctxt->d & DstMask) == ImplicitOps)
3763 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
3764 /* optimisation - avoid slow emulated read if Mov */
3765 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
3766 &ctxt->dst.val, ctxt->dst.bytes);
3767 if (rc != X86EMUL_CONTINUE)
3770 ctxt->dst.orig_val = ctxt->dst.val;
3774 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
3775 rc = emulator_check_intercept(ctxt, ctxt->intercept,
3776 X86_ICPT_POST_MEMACCESS);
3777 if (rc != X86EMUL_CONTINUE)
3781 if (ctxt->execute) {
3782 rc = ctxt->execute(ctxt);
3783 if (rc != X86EMUL_CONTINUE)
3792 case 0x06: /* push es */
3793 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3795 case 0x07: /* pop es */
3796 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3798 case 0x0e: /* push cs */
3799 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3801 case 0x16: /* push ss */
3802 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3804 case 0x17: /* pop ss */
3805 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3807 case 0x1e: /* push ds */
3808 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3810 case 0x1f: /* pop ds */
3811 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3813 case 0x40 ... 0x47: /* inc r16/r32 */
3814 emulate_1op(ctxt, "inc");
3816 case 0x48 ... 0x4f: /* dec r16/r32 */
3817 emulate_1op(ctxt, "dec");
3819 case 0x63: /* movsxd */
3820 if (ctxt->mode != X86EMUL_MODE_PROT64)
3821 goto cannot_emulate;
3822 ctxt->dst.val = (s32) ctxt->src.val;
3824 case 0x6c: /* insb */
3825 case 0x6d: /* insw/insd */
3826 ctxt->src.val = ctxt->regs[VCPU_REGS_RDX];
3828 case 0x6e: /* outsb */
3829 case 0x6f: /* outsw/outsd */
3830 ctxt->dst.val = ctxt->regs[VCPU_REGS_RDX];
3833 case 0x70 ... 0x7f: /* jcc (short) */
3834 if (test_cc(ctxt->b, ctxt->eflags))
3835 jmp_rel(ctxt, ctxt->src.val);
3837 case 0x8d: /* lea r16/r32, m */
3838 ctxt->dst.val = ctxt->src.addr.mem.ea;
3840 case 0x8f: /* pop (sole member of Grp1a) */
3841 rc = em_grp1a(ctxt);
3843 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3844 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
3848 case 0x98: /* cbw/cwde/cdqe */
3849 switch (ctxt->op_bytes) {
3850 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
3851 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
3852 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
3858 case 0xc4: /* les */
3859 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3861 case 0xc5: /* lds */
3862 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3864 case 0xcc: /* int3 */
3865 rc = emulate_int(ctxt, 3);
3867 case 0xcd: /* int n */
3868 rc = emulate_int(ctxt, ctxt->src.val);
3870 case 0xce: /* into */
3871 if (ctxt->eflags & EFLG_OF)
3872 rc = emulate_int(ctxt, 4);
3874 case 0xd0 ... 0xd1: /* Grp2 */
3877 case 0xd2 ... 0xd3: /* Grp2 */
3878 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
3881 case 0xe4: /* inb */
3884 case 0xe6: /* outb */
3885 case 0xe7: /* out */
3887 case 0xe8: /* call (near) */ {
3888 long int rel = ctxt->src.val;
3889 ctxt->src.val = (unsigned long) ctxt->_eip;
3894 case 0xe9: /* jmp rel */
3895 case 0xeb: /* jmp rel short */
3896 jmp_rel(ctxt, ctxt->src.val);
3897 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3899 case 0xec: /* in al,dx */
3900 case 0xed: /* in (e/r)ax,dx */
3902 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3904 goto done; /* IO is needed */
3906 case 0xee: /* out dx,al */
3907 case 0xef: /* out dx,(e/r)ax */
3909 ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3911 ctxt->dst.type = OP_NONE; /* Disable writeback. */
3913 case 0xf4: /* hlt */
3914 ctxt->ops->halt(ctxt);
3916 case 0xf5: /* cmc */
3917 /* complement carry flag from eflags reg */
3918 ctxt->eflags ^= EFLG_CF;
3920 case 0xf6 ... 0xf7: /* Grp3 */
3923 case 0xf8: /* clc */
3924 ctxt->eflags &= ~EFLG_CF;
3926 case 0xf9: /* stc */
3927 ctxt->eflags |= EFLG_CF;
3929 case 0xfc: /* cld */
3930 ctxt->eflags &= ~EFLG_DF;
3932 case 0xfd: /* std */
3933 ctxt->eflags |= EFLG_DF;
3935 case 0xfe: /* Grp4 */
3936 rc = em_grp45(ctxt);
3938 case 0xff: /* Grp5 */
3939 rc = em_grp45(ctxt);
3942 goto cannot_emulate;
3945 if (rc != X86EMUL_CONTINUE)
3949 rc = writeback(ctxt);
3950 if (rc != X86EMUL_CONTINUE)
3954 * restore dst type in case the decoding will be reused
3955 * (happens for string instruction )
3957 ctxt->dst.type = saved_dst_type;
3959 if ((ctxt->d & SrcMask) == SrcSI)
3960 string_addr_inc(ctxt, seg_override(ctxt),
3961 VCPU_REGS_RSI, &ctxt->src);
3963 if ((ctxt->d & DstMask) == DstDI)
3964 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
3967 if (ctxt->rep_prefix && (ctxt->d & String)) {
3968 struct read_cache *r = &ctxt->io_read;
3969 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
3971 if (!string_insn_completed(ctxt)) {
3973 * Re-enter guest when pio read ahead buffer is empty
3974 * or, if it is not used, after each 1024 iteration.
3976 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
3977 (r->end == 0 || r->end != r->pos)) {
3979 * Reset read cache. Usually happens before
3980 * decode, but since instruction is restarted
3981 * we have to do it here.
3983 ctxt->mem_read.end = 0;
3984 return EMULATION_RESTART;
3986 goto done; /* skip rip writeback */
3990 ctxt->eip = ctxt->_eip;
3993 if (rc == X86EMUL_PROPAGATE_FAULT)
3994 ctxt->have_exception = true;
3995 if (rc == X86EMUL_INTERCEPTED)
3996 return EMULATION_INTERCEPTED;
3998 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4002 case 0x09: /* wbinvd */
4003 (ctxt->ops->wbinvd)(ctxt);
4005 case 0x08: /* invd */
4006 case 0x0d: /* GrpP (prefetch) */
4007 case 0x18: /* Grp16 (prefetch/nop) */
4009 case 0x20: /* mov cr, reg */
4010 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4012 case 0x21: /* mov from dr to reg */
4013 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4015 case 0x22: /* mov reg, cr */
4016 if (ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) {
4017 emulate_gp(ctxt, 0);
4018 rc = X86EMUL_PROPAGATE_FAULT;
4021 ctxt->dst.type = OP_NONE;
4023 case 0x23: /* mov from reg to dr */
4024 if (ops->set_dr(ctxt, ctxt->modrm_reg, ctxt->src.val &
4025 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4026 ~0ULL : ~0U)) < 0) {
4027 /* #UD condition is already handled by the code above */
4028 emulate_gp(ctxt, 0);
4029 rc = X86EMUL_PROPAGATE_FAULT;
4033 ctxt->dst.type = OP_NONE; /* no writeback */
4037 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
4038 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
4039 if (ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) {
4040 emulate_gp(ctxt, 0);
4041 rc = X86EMUL_PROPAGATE_FAULT;
4044 rc = X86EMUL_CONTINUE;
4048 if (ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) {
4049 emulate_gp(ctxt, 0);
4050 rc = X86EMUL_PROPAGATE_FAULT;
4053 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
4054 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
4056 rc = X86EMUL_CONTINUE;
4058 case 0x40 ... 0x4f: /* cmov */
4059 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4060 if (!test_cc(ctxt->b, ctxt->eflags))
4061 ctxt->dst.type = OP_NONE; /* no writeback */
4063 case 0x80 ... 0x8f: /* jnz rel, etc*/
4064 if (test_cc(ctxt->b, ctxt->eflags))
4065 jmp_rel(ctxt, ctxt->src.val);
4067 case 0x90 ... 0x9f: /* setcc r/m8 */
4068 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4070 case 0xa0: /* push fs */
4071 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4073 case 0xa1: /* pop fs */
4074 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4078 ctxt->dst.type = OP_NONE;
4079 /* only subword offset */
4080 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
4081 emulate_2op_SrcV_nobyte(ctxt, "bt");
4083 case 0xa4: /* shld imm8, r, r/m */
4084 case 0xa5: /* shld cl, r, r/m */
4085 emulate_2op_cl(ctxt, "shld");
4087 case 0xa8: /* push gs */
4088 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4090 case 0xa9: /* pop gs */
4091 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4095 emulate_2op_SrcV_nobyte(ctxt, "bts");
4097 case 0xac: /* shrd imm8, r, r/m */
4098 case 0xad: /* shrd cl, r, r/m */
4099 emulate_2op_cl(ctxt, "shrd");
4101 case 0xae: /* clflush */
4103 case 0xb0 ... 0xb1: /* cmpxchg */
4105 * Save real source value, then compare EAX against
4108 ctxt->src.orig_val = ctxt->src.val;
4109 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
4110 emulate_2op_SrcV(ctxt, "cmp");
4111 if (ctxt->eflags & EFLG_ZF) {
4112 /* Success: write back to memory. */
4113 ctxt->dst.val = ctxt->src.orig_val;
4115 /* Failure: write the value we saw to EAX. */
4116 ctxt->dst.type = OP_REG;
4117 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
4120 case 0xb2: /* lss */
4121 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4125 emulate_2op_SrcV_nobyte(ctxt, "btr");
4127 case 0xb4: /* lfs */
4128 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4130 case 0xb5: /* lgs */
4131 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4133 case 0xb6 ... 0xb7: /* movzx */
4134 ctxt->dst.bytes = ctxt->op_bytes;
4135 ctxt->dst.val = (ctxt->d & ByteOp) ? (u8) ctxt->src.val
4136 : (u16) ctxt->src.val;
4138 case 0xba: /* Grp8 */
4139 switch (ctxt->modrm_reg & 3) {
4152 emulate_2op_SrcV_nobyte(ctxt, "btc");
4154 case 0xbc: { /* bsf */
4156 __asm__ ("bsf %2, %0; setz %1"
4157 : "=r"(ctxt->dst.val), "=q"(zf)
4158 : "r"(ctxt->src.val));
4159 ctxt->eflags &= ~X86_EFLAGS_ZF;
4161 ctxt->eflags |= X86_EFLAGS_ZF;
4162 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4166 case 0xbd: { /* bsr */
4168 __asm__ ("bsr %2, %0; setz %1"
4169 : "=r"(ctxt->dst.val), "=q"(zf)
4170 : "r"(ctxt->src.val));
4171 ctxt->eflags &= ~X86_EFLAGS_ZF;
4173 ctxt->eflags |= X86_EFLAGS_ZF;
4174 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4178 case 0xbe ... 0xbf: /* movsx */
4179 ctxt->dst.bytes = ctxt->op_bytes;
4180 ctxt->dst.val = (ctxt->d & ByteOp) ? (s8) ctxt->src.val :
4181 (s16) ctxt->src.val;
4183 case 0xc0 ... 0xc1: /* xadd */
4184 emulate_2op_SrcV(ctxt, "add");
4185 /* Write back the register source. */
4186 ctxt->src.val = ctxt->dst.orig_val;
4187 write_register_operand(&ctxt->src);
4189 case 0xc3: /* movnti */
4190 ctxt->dst.bytes = ctxt->op_bytes;
4191 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4192 (u64) ctxt->src.val;
4194 case 0xc7: /* Grp9 (cmpxchg8b) */
4198 goto cannot_emulate;
4201 if (rc != X86EMUL_CONTINUE)
4207 return EMULATION_FAILED;