1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstDX (8<<1) /* Destination is in DX register */
51 #define DstMask (0xf<<1)
52 /* Source operand type. */
53 #define SrcNone (0<<5) /* No source operand. */
54 #define SrcReg (1<<5) /* Register operand. */
55 #define SrcMem (2<<5) /* Memory operand. */
56 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
57 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
58 #define SrcImm (5<<5) /* Immediate operand. */
59 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
60 #define SrcOne (7<<5) /* Implied '1' */
61 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
62 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
63 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
64 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
65 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
66 #define SrcAcc (0xd<<5) /* Source Accumulator */
67 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
68 #define SrcDX (0xf<<5) /* Source is in DX register */
69 #define SrcMask (0xf<<5)
70 /* Generic ModRM decode. */
72 /* Destination is only written; never read. */
75 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
76 #define String (1<<13) /* String instruction (rep capable) */
77 #define Stack (1<<14) /* Stack instruction (push/pop) */
78 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
79 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
80 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
81 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
82 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
83 #define Sse (1<<18) /* SSE Vector instruction */
85 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
86 #define VendorSpecific (1<<22) /* Vendor specific instruction */
87 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89 #define Undefined (1<<25) /* No Such Instruction */
90 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
91 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
93 /* Source 2 operand type */
94 #define Src2None (0<<29)
95 #define Src2CL (1<<29)
96 #define Src2ImmByte (2<<29)
97 #define Src2One (3<<29)
98 #define Src2Imm (4<<29)
99 #define Src2Mask (7<<29)
101 #define X2(x...) x, x
102 #define X3(x...) X2(x), x
103 #define X4(x...) X2(x), X2(x)
104 #define X5(x...) X4(x), x
105 #define X6(x...) X4(x), X2(x)
106 #define X7(x...) X4(x), X3(x)
107 #define X8(x...) X4(x), X4(x)
108 #define X16(x...) X8(x), X8(x)
114 int (*execute)(struct x86_emulate_ctxt *ctxt);
115 struct opcode *group;
116 struct group_dual *gdual;
117 struct gprefix *gprefix;
119 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
123 struct opcode mod012[8];
124 struct opcode mod3[8];
128 struct opcode pfx_no;
129 struct opcode pfx_66;
130 struct opcode pfx_f2;
131 struct opcode pfx_f3;
134 /* EFLAGS bit definitions. */
135 #define EFLG_ID (1<<21)
136 #define EFLG_VIP (1<<20)
137 #define EFLG_VIF (1<<19)
138 #define EFLG_AC (1<<18)
139 #define EFLG_VM (1<<17)
140 #define EFLG_RF (1<<16)
141 #define EFLG_IOPL (3<<12)
142 #define EFLG_NT (1<<14)
143 #define EFLG_OF (1<<11)
144 #define EFLG_DF (1<<10)
145 #define EFLG_IF (1<<9)
146 #define EFLG_TF (1<<8)
147 #define EFLG_SF (1<<7)
148 #define EFLG_ZF (1<<6)
149 #define EFLG_AF (1<<4)
150 #define EFLG_PF (1<<2)
151 #define EFLG_CF (1<<0)
153 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
154 #define EFLG_RESERVED_ONE_MASK 2
157 * Instruction emulation:
158 * Most instructions are emulated directly via a fragment of inline assembly
159 * code. This allows us to save/restore EFLAGS and thus very easily pick up
160 * any modified flags.
163 #if defined(CONFIG_X86_64)
164 #define _LO32 "k" /* force 32-bit operand */
165 #define _STK "%%rsp" /* stack pointer */
166 #elif defined(__i386__)
167 #define _LO32 "" /* force 32-bit operand */
168 #define _STK "%%esp" /* stack pointer */
172 * These EFLAGS bits are restored from saved value during emulation, and
173 * any changes are written back to the saved value after emulation.
175 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
177 /* Before executing instruction: restore necessary bits in EFLAGS. */
178 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
179 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
180 "movl %"_sav",%"_LO32 _tmp"; " \
183 "movl %"_msk",%"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
186 "notl %"_LO32 _tmp"; " \
187 "andl %"_LO32 _tmp",("_STK"); " \
188 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
190 "orl %"_LO32 _tmp",("_STK"); " \
194 /* After executing instruction: write-back necessary bits in EFLAGS. */
195 #define _POST_EFLAGS(_sav, _msk, _tmp) \
196 /* _sav |= EFLAGS & _msk; */ \
199 "andl %"_msk",%"_LO32 _tmp"; " \
200 "orl %"_LO32 _tmp",%"_sav"; "
208 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
210 __asm__ __volatile__ ( \
211 _PRE_EFLAGS("0", "4", "2") \
212 _op _suffix " %"_x"3,%1; " \
213 _POST_EFLAGS("0", "4", "2") \
214 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
216 : _y ((_src).val), "i" (EFLAGS_MASK)); \
220 /* Raw emulation: instruction has two explicit operands. */
221 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
223 unsigned long _tmp; \
225 switch ((_dst).bytes) { \
227 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
230 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
233 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
238 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
240 unsigned long _tmp; \
241 switch ((_dst).bytes) { \
243 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
246 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
247 _wx, _wy, _lx, _ly, _qx, _qy); \
252 /* Source operand is byte-sized and may be restricted to just %cl. */
253 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
254 __emulate_2op(_op, _src, _dst, _eflags, \
255 "b", "c", "b", "c", "b", "c", "b", "c")
257 /* Source operand is byte, word, long or quad sized. */
258 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
259 __emulate_2op(_op, _src, _dst, _eflags, \
260 "b", "q", "w", "r", _LO32, "r", "", "r")
262 /* Source operand is word, long or quad sized. */
263 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
264 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
265 "w", "r", _LO32, "r", "", "r")
267 /* Instruction has three operands and one operand is stored in ECX register */
268 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
270 unsigned long _tmp; \
271 _type _clv = (_cl).val; \
272 _type _srcv = (_src).val; \
273 _type _dstv = (_dst).val; \
275 __asm__ __volatile__ ( \
276 _PRE_EFLAGS("0", "5", "2") \
277 _op _suffix " %4,%1 \n" \
278 _POST_EFLAGS("0", "5", "2") \
279 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
280 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
283 (_cl).val = (unsigned long) _clv; \
284 (_src).val = (unsigned long) _srcv; \
285 (_dst).val = (unsigned long) _dstv; \
288 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
290 switch ((_dst).bytes) { \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "w", unsigned short); \
296 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "l", unsigned int); \
300 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
301 "q", unsigned long)); \
306 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
308 unsigned long _tmp; \
310 __asm__ __volatile__ ( \
311 _PRE_EFLAGS("0", "3", "2") \
312 _op _suffix " %1; " \
313 _POST_EFLAGS("0", "3", "2") \
314 : "=m" (_eflags), "+m" ((_dst).val), \
316 : "i" (EFLAGS_MASK)); \
319 /* Instruction has only one explicit operand (no source operand). */
320 #define emulate_1op(_op, _dst, _eflags) \
322 switch ((_dst).bytes) { \
323 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
324 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
325 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
326 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
330 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
332 unsigned long _tmp; \
334 __asm__ __volatile__ ( \
335 _PRE_EFLAGS("0", "4", "1") \
336 _op _suffix " %5; " \
337 _POST_EFLAGS("0", "4", "1") \
338 : "=m" (_eflags), "=&r" (_tmp), \
339 "+a" (_rax), "+d" (_rdx) \
340 : "i" (EFLAGS_MASK), "m" ((_src).val), \
341 "a" (_rax), "d" (_rdx)); \
344 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "5", "1") \
351 _op _suffix " %6; " \
353 _POST_EFLAGS("0", "5", "1") \
354 ".pushsection .fixup,\"ax\" \n\t" \
355 "3: movb $1, %4 \n\t" \
358 _ASM_EXTABLE(1b, 3b) \
359 : "=m" (_eflags), "=&r" (_tmp), \
360 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
361 : "i" (EFLAGS_MASK), "m" ((_src).val), \
362 "a" (_rax), "d" (_rdx)); \
365 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
366 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
368 switch((_src).bytes) { \
370 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
374 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
378 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
382 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
388 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
390 switch((_src).bytes) { \
392 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
393 _eflags, "b", _ex); \
396 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
397 _eflags, "w", _ex); \
400 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
401 _eflags, "l", _ex); \
404 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
405 _eflags, "q", _ex)); \
410 /* Fetch next part of the instruction being emulated. */
411 #define insn_fetch(_type, _size, _eip) \
412 ({ unsigned long _x; \
413 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
414 if (rc != X86EMUL_CONTINUE) \
420 #define insn_fetch_arr(_arr, _size, _eip) \
421 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
422 if (rc != X86EMUL_CONTINUE) \
427 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
428 enum x86_intercept intercept,
429 enum x86_intercept_stage stage)
431 struct x86_instruction_info info = {
432 .intercept = intercept,
433 .rep_prefix = ctxt->decode.rep_prefix,
434 .modrm_mod = ctxt->decode.modrm_mod,
435 .modrm_reg = ctxt->decode.modrm_reg,
436 .modrm_rm = ctxt->decode.modrm_rm,
437 .src_val = ctxt->decode.src.val64,
438 .src_bytes = ctxt->decode.src.bytes,
439 .dst_bytes = ctxt->decode.dst.bytes,
440 .ad_bytes = ctxt->decode.ad_bytes,
441 .next_rip = ctxt->eip,
444 return ctxt->ops->intercept(ctxt, &info, stage);
447 static inline unsigned long ad_mask(struct decode_cache *c)
449 return (1UL << (c->ad_bytes << 3)) - 1;
452 /* Access/update address held in a register, based on addressing mode. */
453 static inline unsigned long
454 address_mask(struct decode_cache *c, unsigned long reg)
456 if (c->ad_bytes == sizeof(unsigned long))
459 return reg & ad_mask(c);
462 static inline unsigned long
463 register_address(struct decode_cache *c, unsigned long reg)
465 return address_mask(c, reg);
469 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
471 if (c->ad_bytes == sizeof(unsigned long))
474 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
477 static inline void jmp_rel(struct decode_cache *c, int rel)
479 register_address_increment(c, &c->eip, rel);
482 static u32 desc_limit_scaled(struct desc_struct *desc)
484 u32 limit = get_desc_limit(desc);
486 return desc->g ? (limit << 12) | 0xfff : limit;
489 static void set_seg_override(struct decode_cache *c, int seg)
491 c->has_seg_override = true;
492 c->seg_override = seg;
495 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
496 struct x86_emulate_ops *ops, int seg)
498 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
501 return ops->get_cached_segment_base(ctxt, seg);
504 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
505 struct decode_cache *c)
507 if (!c->has_seg_override)
510 return c->seg_override;
513 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
514 u32 error, bool valid)
516 ctxt->exception.vector = vec;
517 ctxt->exception.error_code = error;
518 ctxt->exception.error_code_valid = valid;
519 return X86EMUL_PROPAGATE_FAULT;
522 static int emulate_db(struct x86_emulate_ctxt *ctxt)
524 return emulate_exception(ctxt, DB_VECTOR, 0, false);
527 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
529 return emulate_exception(ctxt, GP_VECTOR, err, true);
532 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
534 return emulate_exception(ctxt, SS_VECTOR, err, true);
537 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
539 return emulate_exception(ctxt, UD_VECTOR, 0, false);
542 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, TS_VECTOR, err, true);
547 static int emulate_de(struct x86_emulate_ctxt *ctxt)
549 return emulate_exception(ctxt, DE_VECTOR, 0, false);
552 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
554 return emulate_exception(ctxt, NM_VECTOR, 0, false);
557 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
560 struct desc_struct desc;
562 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
566 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
571 struct desc_struct desc;
573 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
574 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
577 static int __linearize(struct x86_emulate_ctxt *ctxt,
578 struct segmented_address addr,
579 unsigned size, bool write, bool fetch,
582 struct decode_cache *c = &ctxt->decode;
583 struct desc_struct desc;
590 la = seg_base(ctxt, ctxt->ops, addr.seg) + addr.ea;
591 switch (ctxt->mode) {
592 case X86EMUL_MODE_REAL:
594 case X86EMUL_MODE_PROT64:
595 if (((signed long)la << 16) >> 16 != la)
596 return emulate_gp(ctxt, 0);
599 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
603 /* code segment or read-only data segment */
604 if (((desc.type & 8) || !(desc.type & 2)) && write)
606 /* unreadable code segment */
607 if (!fetch && (desc.type & 8) && !(desc.type & 2))
609 lim = desc_limit_scaled(&desc);
610 if ((desc.type & 8) || !(desc.type & 4)) {
611 /* expand-up segment */
612 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
615 /* exapand-down segment */
616 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
618 lim = desc.d ? 0xffffffff : 0xffff;
619 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
622 cpl = ctxt->ops->cpl(ctxt);
625 if (!(desc.type & 8)) {
629 } else if ((desc.type & 8) && !(desc.type & 4)) {
630 /* nonconforming code segment */
633 } else if ((desc.type & 8) && (desc.type & 4)) {
634 /* conforming code segment */
640 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
643 return X86EMUL_CONTINUE;
645 if (addr.seg == VCPU_SREG_SS)
646 return emulate_ss(ctxt, addr.seg);
648 return emulate_gp(ctxt, addr.seg);
651 static int linearize(struct x86_emulate_ctxt *ctxt,
652 struct segmented_address addr,
653 unsigned size, bool write,
656 return __linearize(ctxt, addr, size, write, false, linear);
660 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
661 struct segmented_address addr,
668 rc = linearize(ctxt, addr, size, false, &linear);
669 if (rc != X86EMUL_CONTINUE)
671 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
674 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
675 struct x86_emulate_ops *ops,
676 unsigned long eip, u8 *dest)
678 struct fetch_cache *fc = &ctxt->decode.fetch;
682 if (eip == fc->end) {
683 unsigned long linear;
684 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
685 cur_size = fc->end - fc->start;
686 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
687 rc = __linearize(ctxt, addr, size, false, true, &linear);
688 if (rc != X86EMUL_CONTINUE)
690 rc = ops->fetch(ctxt, linear, fc->data + cur_size,
691 size, &ctxt->exception);
692 if (rc != X86EMUL_CONTINUE)
696 *dest = fc->data[eip - fc->start];
697 return X86EMUL_CONTINUE;
700 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
701 struct x86_emulate_ops *ops,
702 unsigned long eip, void *dest, unsigned size)
706 /* x86 instructions are limited to 15 bytes. */
707 if (eip + size - ctxt->eip > 15)
708 return X86EMUL_UNHANDLEABLE;
710 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
711 if (rc != X86EMUL_CONTINUE)
714 return X86EMUL_CONTINUE;
718 * Given the 'reg' portion of a ModRM byte, and a register block, return a
719 * pointer into the block that addresses the relevant register.
720 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
722 static void *decode_register(u8 modrm_reg, unsigned long *regs,
727 p = ®s[modrm_reg];
728 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
729 p = (unsigned char *)®s[modrm_reg & 3] + 1;
733 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
734 struct segmented_address addr,
735 u16 *size, unsigned long *address, int op_bytes)
742 rc = segmented_read_std(ctxt, addr, size, 2);
743 if (rc != X86EMUL_CONTINUE)
746 rc = segmented_read_std(ctxt, addr, address, op_bytes);
750 static int test_cc(unsigned int condition, unsigned int flags)
754 switch ((condition & 15) >> 1) {
756 rc |= (flags & EFLG_OF);
758 case 1: /* b/c/nae */
759 rc |= (flags & EFLG_CF);
762 rc |= (flags & EFLG_ZF);
765 rc |= (flags & (EFLG_CF|EFLG_ZF));
768 rc |= (flags & EFLG_SF);
771 rc |= (flags & EFLG_PF);
774 rc |= (flags & EFLG_ZF);
777 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
781 /* Odd condition identifiers (lsb == 1) have inverted sense. */
782 return (!!rc ^ (condition & 1));
785 static void fetch_register_operand(struct operand *op)
789 op->val = *(u8 *)op->addr.reg;
792 op->val = *(u16 *)op->addr.reg;
795 op->val = *(u32 *)op->addr.reg;
798 op->val = *(u64 *)op->addr.reg;
803 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
805 ctxt->ops->get_fpu(ctxt);
807 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
808 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
809 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
810 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
811 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
812 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
813 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
814 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
816 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
817 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
818 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
819 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
820 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
821 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
822 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
823 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
827 ctxt->ops->put_fpu(ctxt);
830 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
833 ctxt->ops->get_fpu(ctxt);
835 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
836 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
837 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
838 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
839 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
840 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
841 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
842 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
844 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
845 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
846 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
847 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
848 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
849 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
850 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
851 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
855 ctxt->ops->put_fpu(ctxt);
858 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
860 struct decode_cache *c,
863 unsigned reg = c->modrm_reg;
864 int highbyte_regs = c->rex_prefix == 0;
867 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
873 read_sse_reg(ctxt, &op->vec_val, reg);
878 if ((c->d & ByteOp) && !inhibit_bytereg) {
879 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
882 op->addr.reg = decode_register(reg, c->regs, 0);
883 op->bytes = c->op_bytes;
885 fetch_register_operand(op);
886 op->orig_val = op->val;
889 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
890 struct x86_emulate_ops *ops,
893 struct decode_cache *c = &ctxt->decode;
895 int index_reg = 0, base_reg = 0, scale;
896 int rc = X86EMUL_CONTINUE;
900 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
901 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
902 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
905 c->modrm = insn_fetch(u8, 1, c->eip);
906 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
907 c->modrm_reg |= (c->modrm & 0x38) >> 3;
908 c->modrm_rm |= (c->modrm & 0x07);
909 c->modrm_seg = VCPU_SREG_DS;
911 if (c->modrm_mod == 3) {
913 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
914 op->addr.reg = decode_register(c->modrm_rm,
915 c->regs, c->d & ByteOp);
919 op->addr.xmm = c->modrm_rm;
920 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
923 fetch_register_operand(op);
929 if (c->ad_bytes == 2) {
930 unsigned bx = c->regs[VCPU_REGS_RBX];
931 unsigned bp = c->regs[VCPU_REGS_RBP];
932 unsigned si = c->regs[VCPU_REGS_RSI];
933 unsigned di = c->regs[VCPU_REGS_RDI];
935 /* 16-bit ModR/M decode. */
936 switch (c->modrm_mod) {
938 if (c->modrm_rm == 6)
939 modrm_ea += insn_fetch(u16, 2, c->eip);
942 modrm_ea += insn_fetch(s8, 1, c->eip);
945 modrm_ea += insn_fetch(u16, 2, c->eip);
948 switch (c->modrm_rm) {
968 if (c->modrm_mod != 0)
975 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
976 (c->modrm_rm == 6 && c->modrm_mod != 0))
977 c->modrm_seg = VCPU_SREG_SS;
978 modrm_ea = (u16)modrm_ea;
980 /* 32/64-bit ModR/M decode. */
981 if ((c->modrm_rm & 7) == 4) {
982 sib = insn_fetch(u8, 1, c->eip);
983 index_reg |= (sib >> 3) & 7;
987 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
988 modrm_ea += insn_fetch(s32, 4, c->eip);
990 modrm_ea += c->regs[base_reg];
992 modrm_ea += c->regs[index_reg] << scale;
993 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
994 if (ctxt->mode == X86EMUL_MODE_PROT64)
997 modrm_ea += c->regs[c->modrm_rm];
998 switch (c->modrm_mod) {
1000 if (c->modrm_rm == 5)
1001 modrm_ea += insn_fetch(s32, 4, c->eip);
1004 modrm_ea += insn_fetch(s8, 1, c->eip);
1007 modrm_ea += insn_fetch(s32, 4, c->eip);
1011 op->addr.mem.ea = modrm_ea;
1016 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1017 struct x86_emulate_ops *ops,
1020 struct decode_cache *c = &ctxt->decode;
1021 int rc = X86EMUL_CONTINUE;
1024 switch (c->ad_bytes) {
1026 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1029 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1032 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1039 static void fetch_bit_operand(struct decode_cache *c)
1043 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1044 mask = ~(c->dst.bytes * 8 - 1);
1046 if (c->src.bytes == 2)
1047 sv = (s16)c->src.val & (s16)mask;
1048 else if (c->src.bytes == 4)
1049 sv = (s32)c->src.val & (s32)mask;
1051 c->dst.addr.mem.ea += (sv >> 3);
1054 /* only subword offset */
1055 c->src.val &= (c->dst.bytes << 3) - 1;
1058 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1059 struct x86_emulate_ops *ops,
1060 unsigned long addr, void *dest, unsigned size)
1063 struct read_cache *mc = &ctxt->decode.mem_read;
1066 int n = min(size, 8u);
1068 if (mc->pos < mc->end)
1071 rc = ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1073 if (rc != X86EMUL_CONTINUE)
1078 memcpy(dest, mc->data + mc->pos, n);
1083 return X86EMUL_CONTINUE;
1086 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1087 struct segmented_address addr,
1094 rc = linearize(ctxt, addr, size, false, &linear);
1095 if (rc != X86EMUL_CONTINUE)
1097 return read_emulated(ctxt, ctxt->ops, linear, data, size);
1100 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1101 struct segmented_address addr,
1108 rc = linearize(ctxt, addr, size, true, &linear);
1109 if (rc != X86EMUL_CONTINUE)
1111 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1115 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1116 struct segmented_address addr,
1117 const void *orig_data, const void *data,
1123 rc = linearize(ctxt, addr, size, true, &linear);
1124 if (rc != X86EMUL_CONTINUE)
1126 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1127 size, &ctxt->exception);
1130 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1131 struct x86_emulate_ops *ops,
1132 unsigned int size, unsigned short port,
1135 struct read_cache *rc = &ctxt->decode.io_read;
1137 if (rc->pos == rc->end) { /* refill pio read ahead */
1138 struct decode_cache *c = &ctxt->decode;
1139 unsigned int in_page, n;
1140 unsigned int count = c->rep_prefix ?
1141 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1142 in_page = (ctxt->eflags & EFLG_DF) ?
1143 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1144 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1145 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1149 rc->pos = rc->end = 0;
1150 if (!ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1155 memcpy(dest, rc->data + rc->pos, size);
1160 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1161 struct x86_emulate_ops *ops,
1162 u16 selector, struct desc_ptr *dt)
1164 if (selector & 1 << 2) {
1165 struct desc_struct desc;
1168 memset (dt, 0, sizeof *dt);
1169 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1172 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1173 dt->address = get_desc_base(&desc);
1175 ops->get_gdt(ctxt, dt);
1178 /* allowed just for 8 bytes segments */
1179 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1180 struct x86_emulate_ops *ops,
1181 u16 selector, struct desc_struct *desc)
1184 u16 index = selector >> 3;
1188 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1190 if (dt.size < index * 8 + 7)
1191 return emulate_gp(ctxt, selector & 0xfffc);
1192 addr = dt.address + index * 8;
1193 ret = ops->read_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1198 /* allowed just for 8 bytes segments */
1199 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1200 struct x86_emulate_ops *ops,
1201 u16 selector, struct desc_struct *desc)
1204 u16 index = selector >> 3;
1208 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1210 if (dt.size < index * 8 + 7)
1211 return emulate_gp(ctxt, selector & 0xfffc);
1213 addr = dt.address + index * 8;
1214 ret = ops->write_std(ctxt, addr, desc, sizeof *desc, &ctxt->exception);
1219 /* Does not support long mode */
1220 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1221 struct x86_emulate_ops *ops,
1222 u16 selector, int seg)
1224 struct desc_struct seg_desc;
1226 unsigned err_vec = GP_VECTOR;
1228 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1231 memset(&seg_desc, 0, sizeof seg_desc);
1233 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1234 || ctxt->mode == X86EMUL_MODE_REAL) {
1235 /* set real mode segment descriptor */
1236 set_desc_base(&seg_desc, selector << 4);
1237 set_desc_limit(&seg_desc, 0xffff);
1244 /* NULL selector is not valid for TR, CS and SS */
1245 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1249 /* TR should be in GDT only */
1250 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1253 if (null_selector) /* for NULL selector skip all following checks */
1256 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1257 if (ret != X86EMUL_CONTINUE)
1260 err_code = selector & 0xfffc;
1261 err_vec = GP_VECTOR;
1263 /* can't load system descriptor into segment selecor */
1264 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1268 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1274 cpl = ops->cpl(ctxt);
1279 * segment is not a writable data segment or segment
1280 * selector's RPL != CPL or segment selector's RPL != CPL
1282 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1286 if (!(seg_desc.type & 8))
1289 if (seg_desc.type & 4) {
1295 if (rpl > cpl || dpl != cpl)
1298 /* CS(RPL) <- CPL */
1299 selector = (selector & 0xfffc) | cpl;
1302 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1305 case VCPU_SREG_LDTR:
1306 if (seg_desc.s || seg_desc.type != 2)
1309 default: /* DS, ES, FS, or GS */
1311 * segment is not a data or readable code segment or
1312 * ((segment is a data or nonconforming code segment)
1313 * and (both RPL and CPL > DPL))
1315 if ((seg_desc.type & 0xa) == 0x8 ||
1316 (((seg_desc.type & 0xc) != 0xc) &&
1317 (rpl > dpl && cpl > dpl)))
1323 /* mark segment as accessed */
1325 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1326 if (ret != X86EMUL_CONTINUE)
1330 ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1331 return X86EMUL_CONTINUE;
1333 emulate_exception(ctxt, err_vec, err_code, true);
1334 return X86EMUL_PROPAGATE_FAULT;
1337 static void write_register_operand(struct operand *op)
1339 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1340 switch (op->bytes) {
1342 *(u8 *)op->addr.reg = (u8)op->val;
1345 *(u16 *)op->addr.reg = (u16)op->val;
1348 *op->addr.reg = (u32)op->val;
1349 break; /* 64b: zero-extend */
1351 *op->addr.reg = op->val;
1356 static int writeback(struct x86_emulate_ctxt *ctxt)
1359 struct decode_cache *c = &ctxt->decode;
1361 switch (c->dst.type) {
1363 write_register_operand(&c->dst);
1367 rc = segmented_cmpxchg(ctxt,
1373 rc = segmented_write(ctxt,
1377 if (rc != X86EMUL_CONTINUE)
1381 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1389 return X86EMUL_CONTINUE;
1392 static int em_push(struct x86_emulate_ctxt *ctxt)
1394 struct decode_cache *c = &ctxt->decode;
1395 struct segmented_address addr;
1397 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1398 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1399 addr.seg = VCPU_SREG_SS;
1401 /* Disable writeback. */
1402 c->dst.type = OP_NONE;
1403 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1406 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1407 void *dest, int len)
1409 struct decode_cache *c = &ctxt->decode;
1411 struct segmented_address addr;
1413 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1414 addr.seg = VCPU_SREG_SS;
1415 rc = segmented_read(ctxt, addr, dest, len);
1416 if (rc != X86EMUL_CONTINUE)
1419 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1423 static int em_pop(struct x86_emulate_ctxt *ctxt)
1425 struct decode_cache *c = &ctxt->decode;
1427 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1430 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1431 struct x86_emulate_ops *ops,
1432 void *dest, int len)
1435 unsigned long val, change_mask;
1436 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1437 int cpl = ops->cpl(ctxt);
1439 rc = emulate_pop(ctxt, &val, len);
1440 if (rc != X86EMUL_CONTINUE)
1443 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1444 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1446 switch(ctxt->mode) {
1447 case X86EMUL_MODE_PROT64:
1448 case X86EMUL_MODE_PROT32:
1449 case X86EMUL_MODE_PROT16:
1451 change_mask |= EFLG_IOPL;
1453 change_mask |= EFLG_IF;
1455 case X86EMUL_MODE_VM86:
1457 return emulate_gp(ctxt, 0);
1458 change_mask |= EFLG_IF;
1460 default: /* real mode */
1461 change_mask |= (EFLG_IOPL | EFLG_IF);
1465 *(unsigned long *)dest =
1466 (ctxt->eflags & ~change_mask) | (val & change_mask);
1471 static int em_popf(struct x86_emulate_ctxt *ctxt)
1473 struct decode_cache *c = &ctxt->decode;
1475 c->dst.type = OP_REG;
1476 c->dst.addr.reg = &ctxt->eflags;
1477 c->dst.bytes = c->op_bytes;
1478 return emulate_popf(ctxt, ctxt->ops, &c->dst.val, c->op_bytes);
1481 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1482 struct x86_emulate_ops *ops, int seg)
1484 struct decode_cache *c = &ctxt->decode;
1486 c->src.val = get_segment_selector(ctxt, seg);
1488 return em_push(ctxt);
1491 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1492 struct x86_emulate_ops *ops, int seg)
1494 struct decode_cache *c = &ctxt->decode;
1495 unsigned long selector;
1498 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1499 if (rc != X86EMUL_CONTINUE)
1502 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1506 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1508 struct decode_cache *c = &ctxt->decode;
1509 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1510 int rc = X86EMUL_CONTINUE;
1511 int reg = VCPU_REGS_RAX;
1513 while (reg <= VCPU_REGS_RDI) {
1514 (reg == VCPU_REGS_RSP) ?
1515 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1518 if (rc != X86EMUL_CONTINUE)
1527 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1529 struct decode_cache *c = &ctxt->decode;
1531 c->src.val = (unsigned long)ctxt->eflags;
1532 return em_push(ctxt);
1535 static int em_popa(struct x86_emulate_ctxt *ctxt)
1537 struct decode_cache *c = &ctxt->decode;
1538 int rc = X86EMUL_CONTINUE;
1539 int reg = VCPU_REGS_RDI;
1541 while (reg >= VCPU_REGS_RAX) {
1542 if (reg == VCPU_REGS_RSP) {
1543 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1548 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1549 if (rc != X86EMUL_CONTINUE)
1556 int emulate_int_real(struct x86_emulate_ctxt *ctxt,
1557 struct x86_emulate_ops *ops, int irq)
1559 struct decode_cache *c = &ctxt->decode;
1566 /* TODO: Add limit checks */
1567 c->src.val = ctxt->eflags;
1569 if (rc != X86EMUL_CONTINUE)
1572 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1574 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1576 if (rc != X86EMUL_CONTINUE)
1579 c->src.val = c->eip;
1581 if (rc != X86EMUL_CONTINUE)
1584 ops->get_idt(ctxt, &dt);
1586 eip_addr = dt.address + (irq << 2);
1587 cs_addr = dt.address + (irq << 2) + 2;
1589 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1590 if (rc != X86EMUL_CONTINUE)
1593 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1594 if (rc != X86EMUL_CONTINUE)
1597 rc = load_segment_descriptor(ctxt, ops, cs, VCPU_SREG_CS);
1598 if (rc != X86EMUL_CONTINUE)
1606 static int emulate_int(struct x86_emulate_ctxt *ctxt,
1607 struct x86_emulate_ops *ops, int irq)
1609 switch(ctxt->mode) {
1610 case X86EMUL_MODE_REAL:
1611 return emulate_int_real(ctxt, ops, irq);
1612 case X86EMUL_MODE_VM86:
1613 case X86EMUL_MODE_PROT16:
1614 case X86EMUL_MODE_PROT32:
1615 case X86EMUL_MODE_PROT64:
1617 /* Protected mode interrupts unimplemented yet */
1618 return X86EMUL_UNHANDLEABLE;
1622 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1623 struct x86_emulate_ops *ops)
1625 struct decode_cache *c = &ctxt->decode;
1626 int rc = X86EMUL_CONTINUE;
1627 unsigned long temp_eip = 0;
1628 unsigned long temp_eflags = 0;
1629 unsigned long cs = 0;
1630 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1631 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1632 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1633 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1635 /* TODO: Add stack limit check */
1637 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1639 if (rc != X86EMUL_CONTINUE)
1642 if (temp_eip & ~0xffff)
1643 return emulate_gp(ctxt, 0);
1645 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1647 if (rc != X86EMUL_CONTINUE)
1650 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1652 if (rc != X86EMUL_CONTINUE)
1655 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1657 if (rc != X86EMUL_CONTINUE)
1663 if (c->op_bytes == 4)
1664 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1665 else if (c->op_bytes == 2) {
1666 ctxt->eflags &= ~0xffff;
1667 ctxt->eflags |= temp_eflags;
1670 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1671 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1676 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1677 struct x86_emulate_ops* ops)
1679 switch(ctxt->mode) {
1680 case X86EMUL_MODE_REAL:
1681 return emulate_iret_real(ctxt, ops);
1682 case X86EMUL_MODE_VM86:
1683 case X86EMUL_MODE_PROT16:
1684 case X86EMUL_MODE_PROT32:
1685 case X86EMUL_MODE_PROT64:
1687 /* iret from protected mode unimplemented yet */
1688 return X86EMUL_UNHANDLEABLE;
1692 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1694 struct decode_cache *c = &ctxt->decode;
1698 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1700 rc = load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS);
1701 if (rc != X86EMUL_CONTINUE)
1705 memcpy(&c->eip, c->src.valptr, c->op_bytes);
1706 return X86EMUL_CONTINUE;
1709 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1711 struct decode_cache *c = &ctxt->decode;
1713 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1716 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1718 struct decode_cache *c = &ctxt->decode;
1719 switch (c->modrm_reg) {
1721 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1724 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1727 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1730 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1732 case 4: /* sal/shl */
1733 case 6: /* sal/shl */
1734 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1737 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1740 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1743 return X86EMUL_CONTINUE;
1746 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1748 struct decode_cache *c = &ctxt->decode;
1749 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1750 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1753 switch (c->modrm_reg) {
1754 case 0 ... 1: /* test */
1755 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1758 c->dst.val = ~c->dst.val;
1761 emulate_1op("neg", c->dst, ctxt->eflags);
1764 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1767 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1770 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1774 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1778 return X86EMUL_UNHANDLEABLE;
1781 return emulate_de(ctxt);
1782 return X86EMUL_CONTINUE;
1785 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1787 struct decode_cache *c = &ctxt->decode;
1788 int rc = X86EMUL_CONTINUE;
1790 switch (c->modrm_reg) {
1792 emulate_1op("inc", c->dst, ctxt->eflags);
1795 emulate_1op("dec", c->dst, ctxt->eflags);
1797 case 2: /* call near abs */ {
1800 c->eip = c->src.val;
1801 c->src.val = old_eip;
1805 case 4: /* jmp abs */
1806 c->eip = c->src.val;
1808 case 5: /* jmp far */
1809 rc = em_jmp_far(ctxt);
1818 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1820 struct decode_cache *c = &ctxt->decode;
1821 u64 old = c->dst.orig_val64;
1823 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1824 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1825 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1826 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1827 ctxt->eflags &= ~EFLG_ZF;
1829 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1830 (u32) c->regs[VCPU_REGS_RBX];
1832 ctxt->eflags |= EFLG_ZF;
1834 return X86EMUL_CONTINUE;
1837 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1838 struct x86_emulate_ops *ops)
1840 struct decode_cache *c = &ctxt->decode;
1844 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1845 if (rc != X86EMUL_CONTINUE)
1847 if (c->op_bytes == 4)
1848 c->eip = (u32)c->eip;
1849 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1850 if (rc != X86EMUL_CONTINUE)
1852 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1856 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt,
1857 struct x86_emulate_ops *ops, int seg)
1859 struct decode_cache *c = &ctxt->decode;
1863 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1865 rc = load_segment_descriptor(ctxt, ops, sel, seg);
1866 if (rc != X86EMUL_CONTINUE)
1869 c->dst.val = c->src.val;
1874 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1875 struct x86_emulate_ops *ops, struct desc_struct *cs,
1876 struct desc_struct *ss)
1880 memset(cs, 0, sizeof(struct desc_struct));
1881 ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1882 memset(ss, 0, sizeof(struct desc_struct));
1884 cs->l = 0; /* will be adjusted later */
1885 set_desc_base(cs, 0); /* flat segment */
1886 cs->g = 1; /* 4kb granularity */
1887 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1888 cs->type = 0x0b; /* Read, Execute, Accessed */
1890 cs->dpl = 0; /* will be adjusted later */
1894 set_desc_base(ss, 0); /* flat segment */
1895 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1896 ss->g = 1; /* 4kb granularity */
1898 ss->type = 0x03; /* Read/Write, Accessed */
1899 ss->d = 1; /* 32bit stack segment */
1904 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
1906 struct x86_emulate_ops *ops = ctxt->ops;
1907 u32 eax, ebx, ecx, edx;
1910 * syscall should always be enabled in longmode - so only become
1911 * vendor specific (cpuid) if other modes are active...
1913 if (ctxt->mode == X86EMUL_MODE_PROT64)
1918 if (ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx)) {
1920 * Intel ("GenuineIntel")
1921 * remark: Intel CPUs only support "syscall" in 64bit
1922 * longmode. Also an 64bit guest with a
1923 * 32bit compat-app running will #UD !! While this
1924 * behaviour can be fixed (by emulating) into AMD
1925 * response - CPUs of AMD can't behave like Intel.
1927 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
1928 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
1929 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
1932 /* AMD ("AuthenticAMD") */
1933 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
1934 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
1935 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1938 /* AMD ("AMDisbetter!") */
1939 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
1940 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
1941 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
1945 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
1950 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
1952 struct decode_cache *c = &ctxt->decode;
1953 struct desc_struct cs, ss;
1958 /* syscall is not available in real mode */
1959 if (ctxt->mode == X86EMUL_MODE_REAL ||
1960 ctxt->mode == X86EMUL_MODE_VM86)
1961 return emulate_ud(ctxt);
1963 if (!(em_syscall_is_enabled(ctxt)))
1964 return emulate_ud(ctxt);
1966 ops->get_msr(ctxt, MSR_EFER, &efer);
1967 setup_syscalls_segments(ctxt, ops, &cs, &ss);
1969 if (!(efer & EFER_SCE))
1970 return emulate_ud(ctxt);
1972 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1974 cs_sel = (u16)(msr_data & 0xfffc);
1975 ss_sel = (u16)(msr_data + 8);
1977 if (efer & EFER_LMA) {
1981 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1982 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1984 c->regs[VCPU_REGS_RCX] = c->eip;
1985 if (efer & EFER_LMA) {
1986 #ifdef CONFIG_X86_64
1987 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1990 ctxt->mode == X86EMUL_MODE_PROT64 ?
1991 MSR_LSTAR : MSR_CSTAR, &msr_data);
1994 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1995 ctxt->eflags &= ~(msr_data | EFLG_RF);
1999 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2000 c->eip = (u32)msr_data;
2002 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2005 return X86EMUL_CONTINUE;
2009 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2011 struct decode_cache *c = &ctxt->decode;
2012 struct desc_struct cs, ss;
2017 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2018 /* inject #GP if in real mode */
2019 if (ctxt->mode == X86EMUL_MODE_REAL)
2020 return emulate_gp(ctxt, 0);
2022 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2023 * Therefore, we inject an #UD.
2025 if (ctxt->mode == X86EMUL_MODE_PROT64)
2026 return emulate_ud(ctxt);
2028 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2030 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2031 switch (ctxt->mode) {
2032 case X86EMUL_MODE_PROT32:
2033 if ((msr_data & 0xfffc) == 0x0)
2034 return emulate_gp(ctxt, 0);
2036 case X86EMUL_MODE_PROT64:
2037 if (msr_data == 0x0)
2038 return emulate_gp(ctxt, 0);
2042 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2043 cs_sel = (u16)msr_data;
2044 cs_sel &= ~SELECTOR_RPL_MASK;
2045 ss_sel = cs_sel + 8;
2046 ss_sel &= ~SELECTOR_RPL_MASK;
2047 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2052 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2053 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2055 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2058 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2059 c->regs[VCPU_REGS_RSP] = msr_data;
2061 return X86EMUL_CONTINUE;
2065 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2067 struct decode_cache *c = &ctxt->decode;
2068 struct desc_struct cs, ss;
2073 /* inject #GP if in real mode or Virtual 8086 mode */
2074 if (ctxt->mode == X86EMUL_MODE_REAL ||
2075 ctxt->mode == X86EMUL_MODE_VM86)
2076 return emulate_gp(ctxt, 0);
2078 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2080 if ((c->rex_prefix & 0x8) != 0x0)
2081 usermode = X86EMUL_MODE_PROT64;
2083 usermode = X86EMUL_MODE_PROT32;
2087 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2089 case X86EMUL_MODE_PROT32:
2090 cs_sel = (u16)(msr_data + 16);
2091 if ((msr_data & 0xfffc) == 0x0)
2092 return emulate_gp(ctxt, 0);
2093 ss_sel = (u16)(msr_data + 24);
2095 case X86EMUL_MODE_PROT64:
2096 cs_sel = (u16)(msr_data + 32);
2097 if (msr_data == 0x0)
2098 return emulate_gp(ctxt, 0);
2099 ss_sel = cs_sel + 8;
2104 cs_sel |= SELECTOR_RPL_MASK;
2105 ss_sel |= SELECTOR_RPL_MASK;
2107 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2108 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2110 c->eip = c->regs[VCPU_REGS_RDX];
2111 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2113 return X86EMUL_CONTINUE;
2116 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2117 struct x86_emulate_ops *ops)
2120 if (ctxt->mode == X86EMUL_MODE_REAL)
2122 if (ctxt->mode == X86EMUL_MODE_VM86)
2124 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2125 return ops->cpl(ctxt) > iopl;
2128 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2129 struct x86_emulate_ops *ops,
2132 struct desc_struct tr_seg;
2135 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2136 unsigned mask = (1 << len) - 1;
2139 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2142 if (desc_limit_scaled(&tr_seg) < 103)
2144 base = get_desc_base(&tr_seg);
2145 #ifdef CONFIG_X86_64
2146 base |= ((u64)base3) << 32;
2148 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2149 if (r != X86EMUL_CONTINUE)
2151 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2153 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2154 if (r != X86EMUL_CONTINUE)
2156 if ((perm >> bit_idx) & mask)
2161 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2162 struct x86_emulate_ops *ops,
2168 if (emulator_bad_iopl(ctxt, ops))
2169 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2172 ctxt->perm_ok = true;
2177 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2178 struct x86_emulate_ops *ops,
2179 struct tss_segment_16 *tss)
2181 struct decode_cache *c = &ctxt->decode;
2184 tss->flag = ctxt->eflags;
2185 tss->ax = c->regs[VCPU_REGS_RAX];
2186 tss->cx = c->regs[VCPU_REGS_RCX];
2187 tss->dx = c->regs[VCPU_REGS_RDX];
2188 tss->bx = c->regs[VCPU_REGS_RBX];
2189 tss->sp = c->regs[VCPU_REGS_RSP];
2190 tss->bp = c->regs[VCPU_REGS_RBP];
2191 tss->si = c->regs[VCPU_REGS_RSI];
2192 tss->di = c->regs[VCPU_REGS_RDI];
2194 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2195 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2196 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2197 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2198 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2201 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2202 struct x86_emulate_ops *ops,
2203 struct tss_segment_16 *tss)
2205 struct decode_cache *c = &ctxt->decode;
2209 ctxt->eflags = tss->flag | 2;
2210 c->regs[VCPU_REGS_RAX] = tss->ax;
2211 c->regs[VCPU_REGS_RCX] = tss->cx;
2212 c->regs[VCPU_REGS_RDX] = tss->dx;
2213 c->regs[VCPU_REGS_RBX] = tss->bx;
2214 c->regs[VCPU_REGS_RSP] = tss->sp;
2215 c->regs[VCPU_REGS_RBP] = tss->bp;
2216 c->regs[VCPU_REGS_RSI] = tss->si;
2217 c->regs[VCPU_REGS_RDI] = tss->di;
2220 * SDM says that segment selectors are loaded before segment
2223 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2224 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2225 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2226 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2227 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2230 * Now load segment descriptors. If fault happenes at this stage
2231 * it is handled in a context of new task
2233 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2234 if (ret != X86EMUL_CONTINUE)
2236 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2237 if (ret != X86EMUL_CONTINUE)
2239 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2240 if (ret != X86EMUL_CONTINUE)
2242 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2243 if (ret != X86EMUL_CONTINUE)
2245 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2246 if (ret != X86EMUL_CONTINUE)
2249 return X86EMUL_CONTINUE;
2252 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2253 struct x86_emulate_ops *ops,
2254 u16 tss_selector, u16 old_tss_sel,
2255 ulong old_tss_base, struct desc_struct *new_desc)
2257 struct tss_segment_16 tss_seg;
2259 u32 new_tss_base = get_desc_base(new_desc);
2261 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2263 if (ret != X86EMUL_CONTINUE)
2264 /* FIXME: need to provide precise fault address */
2267 save_state_to_tss16(ctxt, ops, &tss_seg);
2269 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2271 if (ret != X86EMUL_CONTINUE)
2272 /* FIXME: need to provide precise fault address */
2275 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2277 if (ret != X86EMUL_CONTINUE)
2278 /* FIXME: need to provide precise fault address */
2281 if (old_tss_sel != 0xffff) {
2282 tss_seg.prev_task_link = old_tss_sel;
2284 ret = ops->write_std(ctxt, new_tss_base,
2285 &tss_seg.prev_task_link,
2286 sizeof tss_seg.prev_task_link,
2288 if (ret != X86EMUL_CONTINUE)
2289 /* FIXME: need to provide precise fault address */
2293 return load_state_from_tss16(ctxt, ops, &tss_seg);
2296 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2297 struct x86_emulate_ops *ops,
2298 struct tss_segment_32 *tss)
2300 struct decode_cache *c = &ctxt->decode;
2302 tss->cr3 = ops->get_cr(ctxt, 3);
2304 tss->eflags = ctxt->eflags;
2305 tss->eax = c->regs[VCPU_REGS_RAX];
2306 tss->ecx = c->regs[VCPU_REGS_RCX];
2307 tss->edx = c->regs[VCPU_REGS_RDX];
2308 tss->ebx = c->regs[VCPU_REGS_RBX];
2309 tss->esp = c->regs[VCPU_REGS_RSP];
2310 tss->ebp = c->regs[VCPU_REGS_RBP];
2311 tss->esi = c->regs[VCPU_REGS_RSI];
2312 tss->edi = c->regs[VCPU_REGS_RDI];
2314 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2315 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2316 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2317 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2318 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2319 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2320 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2323 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2324 struct x86_emulate_ops *ops,
2325 struct tss_segment_32 *tss)
2327 struct decode_cache *c = &ctxt->decode;
2330 if (ops->set_cr(ctxt, 3, tss->cr3))
2331 return emulate_gp(ctxt, 0);
2333 ctxt->eflags = tss->eflags | 2;
2334 c->regs[VCPU_REGS_RAX] = tss->eax;
2335 c->regs[VCPU_REGS_RCX] = tss->ecx;
2336 c->regs[VCPU_REGS_RDX] = tss->edx;
2337 c->regs[VCPU_REGS_RBX] = tss->ebx;
2338 c->regs[VCPU_REGS_RSP] = tss->esp;
2339 c->regs[VCPU_REGS_RBP] = tss->ebp;
2340 c->regs[VCPU_REGS_RSI] = tss->esi;
2341 c->regs[VCPU_REGS_RDI] = tss->edi;
2344 * SDM says that segment selectors are loaded before segment
2347 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2348 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2349 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2350 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2351 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2352 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2353 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2356 * Now load segment descriptors. If fault happenes at this stage
2357 * it is handled in a context of new task
2359 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2360 if (ret != X86EMUL_CONTINUE)
2362 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2363 if (ret != X86EMUL_CONTINUE)
2365 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2366 if (ret != X86EMUL_CONTINUE)
2368 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2369 if (ret != X86EMUL_CONTINUE)
2371 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2372 if (ret != X86EMUL_CONTINUE)
2374 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2375 if (ret != X86EMUL_CONTINUE)
2377 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2378 if (ret != X86EMUL_CONTINUE)
2381 return X86EMUL_CONTINUE;
2384 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2385 struct x86_emulate_ops *ops,
2386 u16 tss_selector, u16 old_tss_sel,
2387 ulong old_tss_base, struct desc_struct *new_desc)
2389 struct tss_segment_32 tss_seg;
2391 u32 new_tss_base = get_desc_base(new_desc);
2393 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2395 if (ret != X86EMUL_CONTINUE)
2396 /* FIXME: need to provide precise fault address */
2399 save_state_to_tss32(ctxt, ops, &tss_seg);
2401 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2403 if (ret != X86EMUL_CONTINUE)
2404 /* FIXME: need to provide precise fault address */
2407 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2409 if (ret != X86EMUL_CONTINUE)
2410 /* FIXME: need to provide precise fault address */
2413 if (old_tss_sel != 0xffff) {
2414 tss_seg.prev_task_link = old_tss_sel;
2416 ret = ops->write_std(ctxt, new_tss_base,
2417 &tss_seg.prev_task_link,
2418 sizeof tss_seg.prev_task_link,
2420 if (ret != X86EMUL_CONTINUE)
2421 /* FIXME: need to provide precise fault address */
2425 return load_state_from_tss32(ctxt, ops, &tss_seg);
2428 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2429 struct x86_emulate_ops *ops,
2430 u16 tss_selector, int reason,
2431 bool has_error_code, u32 error_code)
2433 struct desc_struct curr_tss_desc, next_tss_desc;
2435 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2436 ulong old_tss_base =
2437 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2440 /* FIXME: old_tss_base == ~0 ? */
2442 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2443 if (ret != X86EMUL_CONTINUE)
2445 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2446 if (ret != X86EMUL_CONTINUE)
2449 /* FIXME: check that next_tss_desc is tss */
2451 if (reason != TASK_SWITCH_IRET) {
2452 if ((tss_selector & 3) > next_tss_desc.dpl ||
2453 ops->cpl(ctxt) > next_tss_desc.dpl)
2454 return emulate_gp(ctxt, 0);
2457 desc_limit = desc_limit_scaled(&next_tss_desc);
2458 if (!next_tss_desc.p ||
2459 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2460 desc_limit < 0x2b)) {
2461 emulate_ts(ctxt, tss_selector & 0xfffc);
2462 return X86EMUL_PROPAGATE_FAULT;
2465 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2466 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2467 write_segment_descriptor(ctxt, ops, old_tss_sel,
2471 if (reason == TASK_SWITCH_IRET)
2472 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2474 /* set back link to prev task only if NT bit is set in eflags
2475 note that old_tss_sel is not used afetr this point */
2476 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2477 old_tss_sel = 0xffff;
2479 if (next_tss_desc.type & 8)
2480 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2481 old_tss_base, &next_tss_desc);
2483 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2484 old_tss_base, &next_tss_desc);
2485 if (ret != X86EMUL_CONTINUE)
2488 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2489 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2491 if (reason != TASK_SWITCH_IRET) {
2492 next_tss_desc.type |= (1 << 1); /* set busy flag */
2493 write_segment_descriptor(ctxt, ops, tss_selector,
2497 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2498 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2500 if (has_error_code) {
2501 struct decode_cache *c = &ctxt->decode;
2503 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2505 c->src.val = (unsigned long) error_code;
2506 ret = em_push(ctxt);
2512 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2513 u16 tss_selector, int reason,
2514 bool has_error_code, u32 error_code)
2516 struct x86_emulate_ops *ops = ctxt->ops;
2517 struct decode_cache *c = &ctxt->decode;
2521 c->dst.type = OP_NONE;
2523 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2524 has_error_code, error_code);
2526 if (rc == X86EMUL_CONTINUE)
2529 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2532 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2533 int reg, struct operand *op)
2535 struct decode_cache *c = &ctxt->decode;
2536 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2538 register_address_increment(c, &c->regs[reg], df * op->bytes);
2539 op->addr.mem.ea = register_address(c, c->regs[reg]);
2540 op->addr.mem.seg = seg;
2543 static int em_das(struct x86_emulate_ctxt *ctxt)
2545 struct decode_cache *c = &ctxt->decode;
2547 bool af, cf, old_cf;
2549 cf = ctxt->eflags & X86_EFLAGS_CF;
2555 af = ctxt->eflags & X86_EFLAGS_AF;
2556 if ((al & 0x0f) > 9 || af) {
2558 cf = old_cf | (al >= 250);
2563 if (old_al > 0x99 || old_cf) {
2569 /* Set PF, ZF, SF */
2570 c->src.type = OP_IMM;
2573 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2574 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2576 ctxt->eflags |= X86_EFLAGS_CF;
2578 ctxt->eflags |= X86_EFLAGS_AF;
2579 return X86EMUL_CONTINUE;
2582 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2584 struct decode_cache *c = &ctxt->decode;
2589 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2592 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2593 if (load_segment_descriptor(ctxt, ctxt->ops, sel, VCPU_SREG_CS))
2594 return X86EMUL_CONTINUE;
2597 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2599 c->src.val = old_cs;
2601 if (rc != X86EMUL_CONTINUE)
2604 c->src.val = old_eip;
2605 return em_push(ctxt);
2608 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2610 struct decode_cache *c = &ctxt->decode;
2613 c->dst.type = OP_REG;
2614 c->dst.addr.reg = &c->eip;
2615 c->dst.bytes = c->op_bytes;
2616 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2617 if (rc != X86EMUL_CONTINUE)
2619 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2620 return X86EMUL_CONTINUE;
2623 static int em_add(struct x86_emulate_ctxt *ctxt)
2625 struct decode_cache *c = &ctxt->decode;
2627 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2628 return X86EMUL_CONTINUE;
2631 static int em_or(struct x86_emulate_ctxt *ctxt)
2633 struct decode_cache *c = &ctxt->decode;
2635 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2636 return X86EMUL_CONTINUE;
2639 static int em_adc(struct x86_emulate_ctxt *ctxt)
2641 struct decode_cache *c = &ctxt->decode;
2643 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2644 return X86EMUL_CONTINUE;
2647 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2649 struct decode_cache *c = &ctxt->decode;
2651 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2652 return X86EMUL_CONTINUE;
2655 static int em_and(struct x86_emulate_ctxt *ctxt)
2657 struct decode_cache *c = &ctxt->decode;
2659 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2660 return X86EMUL_CONTINUE;
2663 static int em_sub(struct x86_emulate_ctxt *ctxt)
2665 struct decode_cache *c = &ctxt->decode;
2667 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2668 return X86EMUL_CONTINUE;
2671 static int em_xor(struct x86_emulate_ctxt *ctxt)
2673 struct decode_cache *c = &ctxt->decode;
2675 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2676 return X86EMUL_CONTINUE;
2679 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2681 struct decode_cache *c = &ctxt->decode;
2683 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2684 /* Disable writeback. */
2685 c->dst.type = OP_NONE;
2686 return X86EMUL_CONTINUE;
2689 static int em_imul(struct x86_emulate_ctxt *ctxt)
2691 struct decode_cache *c = &ctxt->decode;
2693 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2694 return X86EMUL_CONTINUE;
2697 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2699 struct decode_cache *c = &ctxt->decode;
2701 c->dst.val = c->src2.val;
2702 return em_imul(ctxt);
2705 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2707 struct decode_cache *c = &ctxt->decode;
2709 c->dst.type = OP_REG;
2710 c->dst.bytes = c->src.bytes;
2711 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2712 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2714 return X86EMUL_CONTINUE;
2717 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2719 struct decode_cache *c = &ctxt->decode;
2722 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2723 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2724 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2725 return X86EMUL_CONTINUE;
2728 static int em_mov(struct x86_emulate_ctxt *ctxt)
2730 struct decode_cache *c = &ctxt->decode;
2731 c->dst.val = c->src.val;
2732 return X86EMUL_CONTINUE;
2735 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2737 struct decode_cache *c = &ctxt->decode;
2738 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2739 return X86EMUL_CONTINUE;
2742 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2744 struct decode_cache *c = &ctxt->decode;
2748 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2749 if (rc == X86EMUL_CONTINUE)
2750 ctxt->ops->invlpg(ctxt, linear);
2751 /* Disable writeback. */
2752 c->dst.type = OP_NONE;
2753 return X86EMUL_CONTINUE;
2756 static int em_clts(struct x86_emulate_ctxt *ctxt)
2760 cr0 = ctxt->ops->get_cr(ctxt, 0);
2762 ctxt->ops->set_cr(ctxt, 0, cr0);
2763 return X86EMUL_CONTINUE;
2766 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2768 struct decode_cache *c = &ctxt->decode;
2771 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2772 return X86EMUL_UNHANDLEABLE;
2774 rc = ctxt->ops->fix_hypercall(ctxt);
2775 if (rc != X86EMUL_CONTINUE)
2778 /* Let the processor re-execute the fixed hypercall */
2780 /* Disable writeback. */
2781 c->dst.type = OP_NONE;
2782 return X86EMUL_CONTINUE;
2785 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2787 struct decode_cache *c = &ctxt->decode;
2788 struct desc_ptr desc_ptr;
2791 rc = read_descriptor(ctxt, c->src.addr.mem,
2792 &desc_ptr.size, &desc_ptr.address,
2794 if (rc != X86EMUL_CONTINUE)
2796 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2797 /* Disable writeback. */
2798 c->dst.type = OP_NONE;
2799 return X86EMUL_CONTINUE;
2802 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2804 struct decode_cache *c = &ctxt->decode;
2807 rc = ctxt->ops->fix_hypercall(ctxt);
2809 /* Disable writeback. */
2810 c->dst.type = OP_NONE;
2814 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2816 struct decode_cache *c = &ctxt->decode;
2817 struct desc_ptr desc_ptr;
2820 rc = read_descriptor(ctxt, c->src.addr.mem,
2821 &desc_ptr.size, &desc_ptr.address,
2823 if (rc != X86EMUL_CONTINUE)
2825 ctxt->ops->set_idt(ctxt, &desc_ptr);
2826 /* Disable writeback. */
2827 c->dst.type = OP_NONE;
2828 return X86EMUL_CONTINUE;
2831 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2833 struct decode_cache *c = &ctxt->decode;
2836 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2837 return X86EMUL_CONTINUE;
2840 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2842 struct decode_cache *c = &ctxt->decode;
2843 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2844 | (c->src.val & 0x0f));
2845 c->dst.type = OP_NONE;
2846 return X86EMUL_CONTINUE;
2849 static bool valid_cr(int nr)
2861 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2863 struct decode_cache *c = &ctxt->decode;
2865 if (!valid_cr(c->modrm_reg))
2866 return emulate_ud(ctxt);
2868 return X86EMUL_CONTINUE;
2871 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2873 struct decode_cache *c = &ctxt->decode;
2874 u64 new_val = c->src.val64;
2875 int cr = c->modrm_reg;
2878 static u64 cr_reserved_bits[] = {
2879 0xffffffff00000000ULL,
2880 0, 0, 0, /* CR3 checked later */
2887 return emulate_ud(ctxt);
2889 if (new_val & cr_reserved_bits[cr])
2890 return emulate_gp(ctxt, 0);
2895 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2896 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2897 return emulate_gp(ctxt, 0);
2899 cr4 = ctxt->ops->get_cr(ctxt, 4);
2900 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2902 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2903 !(cr4 & X86_CR4_PAE))
2904 return emulate_gp(ctxt, 0);
2911 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2912 if (efer & EFER_LMA)
2913 rsvd = CR3_L_MODE_RESERVED_BITS;
2914 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2915 rsvd = CR3_PAE_RESERVED_BITS;
2916 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2917 rsvd = CR3_NONPAE_RESERVED_BITS;
2920 return emulate_gp(ctxt, 0);
2927 cr4 = ctxt->ops->get_cr(ctxt, 4);
2928 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2930 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2931 return emulate_gp(ctxt, 0);
2937 return X86EMUL_CONTINUE;
2940 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2944 ctxt->ops->get_dr(ctxt, 7, &dr7);
2946 /* Check if DR7.Global_Enable is set */
2947 return dr7 & (1 << 13);
2950 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2952 struct decode_cache *c = &ctxt->decode;
2953 int dr = c->modrm_reg;
2957 return emulate_ud(ctxt);
2959 cr4 = ctxt->ops->get_cr(ctxt, 4);
2960 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2961 return emulate_ud(ctxt);
2963 if (check_dr7_gd(ctxt))
2964 return emulate_db(ctxt);
2966 return X86EMUL_CONTINUE;
2969 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2971 struct decode_cache *c = &ctxt->decode;
2972 u64 new_val = c->src.val64;
2973 int dr = c->modrm_reg;
2975 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2976 return emulate_gp(ctxt, 0);
2978 return check_dr_read(ctxt);
2981 static int check_svme(struct x86_emulate_ctxt *ctxt)
2985 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2987 if (!(efer & EFER_SVME))
2988 return emulate_ud(ctxt);
2990 return X86EMUL_CONTINUE;
2993 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2995 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2997 /* Valid physical address? */
2998 if (rax & 0xffff000000000000ULL)
2999 return emulate_gp(ctxt, 0);
3001 return check_svme(ctxt);
3004 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3006 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3008 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3009 return emulate_ud(ctxt);
3011 return X86EMUL_CONTINUE;
3014 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3016 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3017 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
3019 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3021 return emulate_gp(ctxt, 0);
3023 return X86EMUL_CONTINUE;
3026 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3028 struct decode_cache *c = &ctxt->decode;
3030 c->dst.bytes = min(c->dst.bytes, 4u);
3031 if (!emulator_io_permited(ctxt, ctxt->ops, c->src.val, c->dst.bytes))
3032 return emulate_gp(ctxt, 0);
3034 return X86EMUL_CONTINUE;
3037 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3039 struct decode_cache *c = &ctxt->decode;
3041 c->src.bytes = min(c->src.bytes, 4u);
3042 if (!emulator_io_permited(ctxt, ctxt->ops, c->dst.val, c->src.bytes))
3043 return emulate_gp(ctxt, 0);
3045 return X86EMUL_CONTINUE;
3048 #define D(_y) { .flags = (_y) }
3049 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3050 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3051 .check_perm = (_p) }
3053 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3054 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
3055 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
3056 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3057 #define II(_f, _e, _i) \
3058 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3059 #define IIP(_f, _e, _i, _p) \
3060 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3061 .check_perm = (_p) }
3062 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3064 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3065 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3066 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3068 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3069 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3070 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3072 static struct opcode group7_rm1[] = {
3073 DI(SrcNone | ModRM | Priv, monitor),
3074 DI(SrcNone | ModRM | Priv, mwait),
3078 static struct opcode group7_rm3[] = {
3079 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3080 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3081 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3082 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3083 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3084 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3085 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3086 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3089 static struct opcode group7_rm7[] = {
3091 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3095 static struct opcode group1[] = {
3106 static struct opcode group1A[] = {
3107 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3110 static struct opcode group3[] = {
3111 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3112 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3113 X4(D(SrcMem | ModRM)),
3116 static struct opcode group4[] = {
3117 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3121 static struct opcode group5[] = {
3122 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3123 D(SrcMem | ModRM | Stack),
3124 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3125 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3126 D(SrcMem | ModRM | Stack), N,
3129 static struct opcode group6[] = {
3130 DI(ModRM | Prot, sldt),
3131 DI(ModRM | Prot, str),
3132 DI(ModRM | Prot | Priv, lldt),
3133 DI(ModRM | Prot | Priv, ltr),
3137 static struct group_dual group7 = { {
3138 DI(ModRM | Mov | DstMem | Priv, sgdt),
3139 DI(ModRM | Mov | DstMem | Priv, sidt),
3140 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3141 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3142 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3143 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3144 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3146 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3148 N, EXT(0, group7_rm3),
3149 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3150 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3153 static struct opcode group8[] = {
3155 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3156 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3159 static struct group_dual group9 = { {
3160 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3162 N, N, N, N, N, N, N, N,
3165 static struct opcode group11[] = {
3166 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3169 static struct gprefix pfx_0f_6f_0f_7f = {
3170 N, N, N, I(Sse, em_movdqu),
3173 static struct opcode opcode_table[256] = {
3175 I6ALU(Lock, em_add),
3176 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3179 D(ImplicitOps | Stack | No64), N,
3181 I6ALU(Lock, em_adc),
3182 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3184 I6ALU(Lock, em_sbb),
3185 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3187 I6ALU(Lock, em_and), N, N,
3189 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3191 I6ALU(Lock, em_xor), N, N,
3193 I6ALU(0, em_cmp), N, N,
3197 X8(I(SrcReg | Stack, em_push)),
3199 X8(I(DstReg | Stack, em_pop)),
3201 I(ImplicitOps | Stack | No64, em_pusha),
3202 I(ImplicitOps | Stack | No64, em_popa),
3203 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3206 I(SrcImm | Mov | Stack, em_push),
3207 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3208 I(SrcImmByte | Mov | Stack, em_push),
3209 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3210 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3211 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3215 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3216 G(DstMem | SrcImm | ModRM | Group, group1),
3217 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3218 G(DstMem | SrcImmByte | ModRM | Group, group1),
3219 D2bv(DstMem | SrcReg | ModRM), D2bv(DstMem | SrcReg | ModRM | Lock),
3221 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3222 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3223 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3224 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3226 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3228 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3229 I(SrcImmFAddr | No64, em_call_far), N,
3230 II(ImplicitOps | Stack, em_pushf, pushf),
3231 II(ImplicitOps | Stack, em_popf, popf), N, N,
3233 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3234 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3235 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3236 I2bv(SrcSI | DstDI | String, em_cmp),
3238 D2bv(DstAcc | SrcImm),
3239 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3240 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3241 I2bv(SrcAcc | DstDI | String, em_cmp),
3243 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3245 X8(I(DstReg | SrcImm | Mov, em_mov)),
3247 D2bv(DstMem | SrcImmByte | ModRM),
3248 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3249 D(ImplicitOps | Stack),
3250 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3251 G(ByteOp, group11), G(0, group11),
3253 N, N, N, D(ImplicitOps | Stack),
3254 D(ImplicitOps), DI(SrcImmByte, intn),
3255 D(ImplicitOps | No64), DI(ImplicitOps, iret),
3257 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3260 N, N, N, N, N, N, N, N,
3263 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3264 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3266 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3267 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
3268 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3269 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3271 N, DI(ImplicitOps, icebp), N, N,
3272 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3273 G(ByteOp, group3), G(0, group3),
3275 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3276 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3279 static struct opcode twobyte_table[256] = {
3281 G(0, group6), GD(0, &group7), N, N,
3282 N, D(ImplicitOps | VendorSpecific), DI(ImplicitOps | Priv, clts), N,
3283 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3284 N, D(ImplicitOps | ModRM), N, N,
3286 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3288 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3289 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3290 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3291 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3293 N, N, N, N, N, N, N, N,
3295 DI(ImplicitOps | Priv, wrmsr),
3296 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3297 DI(ImplicitOps | Priv, rdmsr),
3298 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3299 D(ImplicitOps | VendorSpecific), D(ImplicitOps | Priv | VendorSpecific),
3301 N, N, N, N, N, N, N, N,
3303 X16(D(DstReg | SrcMem | ModRM | Mov)),
3305 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3310 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3315 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3319 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3321 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3322 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3323 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3324 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3326 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3327 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3328 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3329 D(DstMem | SrcReg | Src2CL | ModRM),
3330 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3332 D2bv(DstMem | SrcReg | ModRM | Lock),
3333 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3334 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3335 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3338 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3339 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3340 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3342 D2bv(DstMem | SrcReg | ModRM | Lock),
3343 N, D(DstMem | SrcReg | ModRM | Mov),
3344 N, N, N, GD(0, &group9),
3345 N, N, N, N, N, N, N, N,
3347 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3349 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3351 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3367 static unsigned imm_size(struct decode_cache *c)
3371 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3377 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3378 unsigned size, bool sign_extension)
3380 struct decode_cache *c = &ctxt->decode;
3381 struct x86_emulate_ops *ops = ctxt->ops;
3382 int rc = X86EMUL_CONTINUE;
3386 op->addr.mem.ea = c->eip;
3387 /* NB. Immediates are sign-extended as necessary. */
3388 switch (op->bytes) {
3390 op->val = insn_fetch(s8, 1, c->eip);
3393 op->val = insn_fetch(s16, 2, c->eip);
3396 op->val = insn_fetch(s32, 4, c->eip);
3399 if (!sign_extension) {
3400 switch (op->bytes) {
3408 op->val &= 0xffffffff;
3417 x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3419 struct x86_emulate_ops *ops = ctxt->ops;
3420 struct decode_cache *c = &ctxt->decode;
3421 int rc = X86EMUL_CONTINUE;
3422 int mode = ctxt->mode;
3423 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3424 bool op_prefix = false;
3425 struct opcode opcode;
3426 struct operand memop = { .type = OP_NONE }, *memopp = NULL;
3429 c->fetch.start = c->eip;
3430 c->fetch.end = c->fetch.start + insn_len;
3432 memcpy(c->fetch.data, insn, insn_len);
3435 case X86EMUL_MODE_REAL:
3436 case X86EMUL_MODE_VM86:
3437 case X86EMUL_MODE_PROT16:
3438 def_op_bytes = def_ad_bytes = 2;
3440 case X86EMUL_MODE_PROT32:
3441 def_op_bytes = def_ad_bytes = 4;
3443 #ifdef CONFIG_X86_64
3444 case X86EMUL_MODE_PROT64:
3453 c->op_bytes = def_op_bytes;
3454 c->ad_bytes = def_ad_bytes;
3456 /* Legacy prefixes. */
3458 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3459 case 0x66: /* operand-size override */
3461 /* switch between 2/4 bytes */
3462 c->op_bytes = def_op_bytes ^ 6;
3464 case 0x67: /* address-size override */
3465 if (mode == X86EMUL_MODE_PROT64)
3466 /* switch between 4/8 bytes */
3467 c->ad_bytes = def_ad_bytes ^ 12;
3469 /* switch between 2/4 bytes */
3470 c->ad_bytes = def_ad_bytes ^ 6;
3472 case 0x26: /* ES override */
3473 case 0x2e: /* CS override */
3474 case 0x36: /* SS override */
3475 case 0x3e: /* DS override */
3476 set_seg_override(c, (c->b >> 3) & 3);
3478 case 0x64: /* FS override */
3479 case 0x65: /* GS override */
3480 set_seg_override(c, c->b & 7);
3482 case 0x40 ... 0x4f: /* REX */
3483 if (mode != X86EMUL_MODE_PROT64)
3485 c->rex_prefix = c->b;
3487 case 0xf0: /* LOCK */
3490 case 0xf2: /* REPNE/REPNZ */
3491 case 0xf3: /* REP/REPE/REPZ */
3492 c->rep_prefix = c->b;
3498 /* Any legacy prefix after a REX prefix nullifies its effect. */
3506 if (c->rex_prefix & 8)
3507 c->op_bytes = 8; /* REX.W */
3509 /* Opcode byte(s). */
3510 opcode = opcode_table[c->b];
3511 /* Two-byte opcode? */
3514 c->b = insn_fetch(u8, 1, c->eip);
3515 opcode = twobyte_table[c->b];
3517 c->d = opcode.flags;
3519 while (c->d & GroupMask) {
3520 switch (c->d & GroupMask) {
3522 c->modrm = insn_fetch(u8, 1, c->eip);
3524 goffset = (c->modrm >> 3) & 7;
3525 opcode = opcode.u.group[goffset];
3528 c->modrm = insn_fetch(u8, 1, c->eip);
3530 goffset = (c->modrm >> 3) & 7;
3531 if ((c->modrm >> 6) == 3)
3532 opcode = opcode.u.gdual->mod3[goffset];
3534 opcode = opcode.u.gdual->mod012[goffset];
3537 goffset = c->modrm & 7;
3538 opcode = opcode.u.group[goffset];
3541 if (c->rep_prefix && op_prefix)
3542 return X86EMUL_UNHANDLEABLE;
3543 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3544 switch (simd_prefix) {
3545 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3546 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3547 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3548 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3552 return X86EMUL_UNHANDLEABLE;
3556 c->d |= opcode.flags;
3559 c->execute = opcode.u.execute;
3560 c->check_perm = opcode.check_perm;
3561 c->intercept = opcode.intercept;
3564 if (c->d == 0 || (c->d & Undefined))
3567 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3570 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3573 if (c->d & Op3264) {
3574 if (mode == X86EMUL_MODE_PROT64)
3583 /* ModRM and SIB bytes. */
3585 rc = decode_modrm(ctxt, ops, &memop);
3586 if (!c->has_seg_override)
3587 set_seg_override(c, c->modrm_seg);
3588 } else if (c->d & MemAbs)
3589 rc = decode_abs(ctxt, ops, &memop);
3590 if (rc != X86EMUL_CONTINUE)
3593 if (!c->has_seg_override)
3594 set_seg_override(c, VCPU_SREG_DS);
3596 memop.addr.mem.seg = seg_override(ctxt, c);
3598 if (memop.type == OP_MEM && c->ad_bytes != 8)
3599 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3602 * Decode and fetch the source operand: register, memory
3605 switch (c->d & SrcMask) {
3609 decode_register_operand(ctxt, &c->src, c, 0);
3618 memop.bytes = (c->d & ByteOp) ? 1 :
3625 rc = decode_imm(ctxt, &c->src, 2, false);
3628 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3631 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3634 rc = decode_imm(ctxt, &c->src, 1, true);
3637 rc = decode_imm(ctxt, &c->src, 1, false);
3640 c->src.type = OP_REG;
3641 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3642 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3643 fetch_register_operand(&c->src);
3650 c->src.type = OP_MEM;
3651 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3652 c->src.addr.mem.ea =
3653 register_address(c, c->regs[VCPU_REGS_RSI]);
3654 c->src.addr.mem.seg = seg_override(ctxt, c);
3658 c->src.type = OP_IMM;
3659 c->src.addr.mem.ea = c->eip;
3660 c->src.bytes = c->op_bytes + 2;
3661 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3664 memop.bytes = c->op_bytes + 2;
3668 c->src.type = OP_REG;
3670 c->src.addr.reg = &c->regs[VCPU_REGS_RDX];
3671 fetch_register_operand(&c->src);
3675 if (rc != X86EMUL_CONTINUE)
3679 * Decode and fetch the second source operand: register, memory
3682 switch (c->d & Src2Mask) {
3687 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3690 rc = decode_imm(ctxt, &c->src2, 1, true);
3697 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3701 if (rc != X86EMUL_CONTINUE)
3704 /* Decode and fetch the destination operand: register or memory. */
3705 switch (c->d & DstMask) {
3707 decode_register_operand(ctxt, &c->dst, c,
3708 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3711 c->dst.type = OP_IMM;
3712 c->dst.addr.mem.ea = c->eip;
3714 c->dst.val = insn_fetch(u8, 1, c->eip);
3720 if ((c->d & DstMask) == DstMem64)
3723 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3725 fetch_bit_operand(c);
3726 c->dst.orig_val = c->dst.val;
3729 c->dst.type = OP_REG;
3730 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3731 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3732 fetch_register_operand(&c->dst);
3733 c->dst.orig_val = c->dst.val;
3736 c->dst.type = OP_MEM;
3737 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3738 c->dst.addr.mem.ea =
3739 register_address(c, c->regs[VCPU_REGS_RDI]);
3740 c->dst.addr.mem.seg = VCPU_SREG_ES;
3744 c->dst.type = OP_REG;
3746 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
3747 fetch_register_operand(&c->dst);
3750 /* Special instructions do their own operand decoding. */
3752 c->dst.type = OP_NONE; /* Disable writeback. */
3757 if (memopp && memopp->type == OP_MEM && c->rip_relative)
3758 memopp->addr.mem.ea += c->eip;
3760 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3763 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3765 struct decode_cache *c = &ctxt->decode;
3767 /* The second termination condition only applies for REPE
3768 * and REPNE. Test if the repeat string operation prefix is
3769 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3770 * corresponding termination condition according to:
3771 * - if REPE/REPZ and ZF = 0 then done
3772 * - if REPNE/REPNZ and ZF = 1 then done
3774 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3775 (c->b == 0xae) || (c->b == 0xaf))
3776 && (((c->rep_prefix == REPE_PREFIX) &&
3777 ((ctxt->eflags & EFLG_ZF) == 0))
3778 || ((c->rep_prefix == REPNE_PREFIX) &&
3779 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3786 x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3788 struct x86_emulate_ops *ops = ctxt->ops;
3790 struct decode_cache *c = &ctxt->decode;
3791 int rc = X86EMUL_CONTINUE;
3792 int saved_dst_type = c->dst.type;
3793 int irq; /* Used for int 3, int, and into */
3795 ctxt->decode.mem_read.pos = 0;
3797 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3798 rc = emulate_ud(ctxt);
3802 /* LOCK prefix is allowed only with some instructions */
3803 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3804 rc = emulate_ud(ctxt);
3808 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3809 rc = emulate_ud(ctxt);
3814 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3815 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3816 rc = emulate_ud(ctxt);
3820 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3821 rc = emulate_nm(ctxt);
3825 if (unlikely(ctxt->guest_mode) && c->intercept) {
3826 rc = emulator_check_intercept(ctxt, c->intercept,
3827 X86_ICPT_PRE_EXCEPT);
3828 if (rc != X86EMUL_CONTINUE)
3832 /* Privileged instruction can be executed only in CPL=0 */
3833 if ((c->d & Priv) && ops->cpl(ctxt)) {
3834 rc = emulate_gp(ctxt, 0);
3838 /* Instruction can only be executed in protected mode */
3839 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3840 rc = emulate_ud(ctxt);
3844 /* Do instruction specific permission checks */
3845 if (c->check_perm) {
3846 rc = c->check_perm(ctxt);
3847 if (rc != X86EMUL_CONTINUE)
3851 if (unlikely(ctxt->guest_mode) && c->intercept) {
3852 rc = emulator_check_intercept(ctxt, c->intercept,
3853 X86_ICPT_POST_EXCEPT);
3854 if (rc != X86EMUL_CONTINUE)
3858 if (c->rep_prefix && (c->d & String)) {
3859 /* All REP prefixes have the same first termination condition */
3860 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3866 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3867 rc = segmented_read(ctxt, c->src.addr.mem,
3868 c->src.valptr, c->src.bytes);
3869 if (rc != X86EMUL_CONTINUE)
3871 c->src.orig_val64 = c->src.val64;
3874 if (c->src2.type == OP_MEM) {
3875 rc = segmented_read(ctxt, c->src2.addr.mem,
3876 &c->src2.val, c->src2.bytes);
3877 if (rc != X86EMUL_CONTINUE)
3881 if ((c->d & DstMask) == ImplicitOps)
3885 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3886 /* optimisation - avoid slow emulated read if Mov */
3887 rc = segmented_read(ctxt, c->dst.addr.mem,
3888 &c->dst.val, c->dst.bytes);
3889 if (rc != X86EMUL_CONTINUE)
3892 c->dst.orig_val = c->dst.val;
3896 if (unlikely(ctxt->guest_mode) && c->intercept) {
3897 rc = emulator_check_intercept(ctxt, c->intercept,
3898 X86_ICPT_POST_MEMACCESS);
3899 if (rc != X86EMUL_CONTINUE)
3904 rc = c->execute(ctxt);
3905 if (rc != X86EMUL_CONTINUE)
3914 case 0x06: /* push es */
3915 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
3917 case 0x07: /* pop es */
3918 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
3920 case 0x0e: /* push cs */
3921 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
3923 case 0x16: /* push ss */
3924 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
3926 case 0x17: /* pop ss */
3927 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
3929 case 0x1e: /* push ds */
3930 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
3932 case 0x1f: /* pop ds */
3933 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
3935 case 0x40 ... 0x47: /* inc r16/r32 */
3936 emulate_1op("inc", c->dst, ctxt->eflags);
3938 case 0x48 ... 0x4f: /* dec r16/r32 */
3939 emulate_1op("dec", c->dst, ctxt->eflags);
3941 case 0x63: /* movsxd */
3942 if (ctxt->mode != X86EMUL_MODE_PROT64)
3943 goto cannot_emulate;
3944 c->dst.val = (s32) c->src.val;
3946 case 0x6c: /* insb */
3947 case 0x6d: /* insw/insd */
3948 c->src.val = c->regs[VCPU_REGS_RDX];
3950 case 0x6e: /* outsb */
3951 case 0x6f: /* outsw/outsd */
3952 c->dst.val = c->regs[VCPU_REGS_RDX];
3955 case 0x70 ... 0x7f: /* jcc (short) */
3956 if (test_cc(c->b, ctxt->eflags))
3957 jmp_rel(c, c->src.val);
3961 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
3963 case 0x86 ... 0x87: /* xchg */
3965 /* Write back the register source. */
3966 c->src.val = c->dst.val;
3967 write_register_operand(&c->src);
3969 * Write back the memory destination with implicit LOCK
3972 c->dst.val = c->src.orig_val;
3975 case 0x8c: /* mov r/m, sreg */
3976 if (c->modrm_reg > VCPU_SREG_GS) {
3977 rc = emulate_ud(ctxt);
3980 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3982 case 0x8d: /* lea r16/r32, m */
3983 c->dst.val = c->src.addr.mem.ea;
3985 case 0x8e: { /* mov seg, r/m16 */
3990 if (c->modrm_reg == VCPU_SREG_CS ||
3991 c->modrm_reg > VCPU_SREG_GS) {
3992 rc = emulate_ud(ctxt);
3996 if (c->modrm_reg == VCPU_SREG_SS)
3997 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3999 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
4001 c->dst.type = OP_NONE; /* Disable writeback. */
4004 case 0x8f: /* pop (sole member of Grp1a) */
4005 rc = em_grp1a(ctxt);
4007 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4008 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
4011 case 0x98: /* cbw/cwde/cdqe */
4012 switch (c->op_bytes) {
4013 case 2: c->dst.val = (s8)c->dst.val; break;
4014 case 4: c->dst.val = (s16)c->dst.val; break;
4015 case 8: c->dst.val = (s32)c->dst.val; break;
4018 case 0xa8 ... 0xa9: /* test ax, imm */
4023 case 0xc3: /* ret */
4024 c->dst.type = OP_REG;
4025 c->dst.addr.reg = &c->eip;
4026 c->dst.bytes = c->op_bytes;
4029 case 0xc4: /* les */
4030 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_ES);
4032 case 0xc5: /* lds */
4033 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_DS);
4035 case 0xcb: /* ret far */
4036 rc = emulate_ret_far(ctxt, ops);
4038 case 0xcc: /* int3 */
4041 case 0xcd: /* int n */
4044 rc = emulate_int(ctxt, ops, irq);
4046 case 0xce: /* into */
4047 if (ctxt->eflags & EFLG_OF) {
4052 case 0xcf: /* iret */
4053 rc = emulate_iret(ctxt, ops);
4055 case 0xd0 ... 0xd1: /* Grp2 */
4058 case 0xd2 ... 0xd3: /* Grp2 */
4059 c->src.val = c->regs[VCPU_REGS_RCX];
4062 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
4063 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4064 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
4065 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
4066 jmp_rel(c, c->src.val);
4068 case 0xe3: /* jcxz/jecxz/jrcxz */
4069 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
4070 jmp_rel(c, c->src.val);
4072 case 0xe4: /* inb */
4075 case 0xe6: /* outb */
4076 case 0xe7: /* out */
4078 case 0xe8: /* call (near) */ {
4079 long int rel = c->src.val;
4080 c->src.val = (unsigned long) c->eip;
4085 case 0xe9: /* jmp rel */
4087 case 0xea: /* jmp far */
4088 rc = em_jmp_far(ctxt);
4091 jmp: /* jmp rel short */
4092 jmp_rel(c, c->src.val);
4093 c->dst.type = OP_NONE; /* Disable writeback. */
4095 case 0xec: /* in al,dx */
4096 case 0xed: /* in (e/r)ax,dx */
4098 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
4100 goto done; /* IO is needed */
4102 case 0xee: /* out dx,al */
4103 case 0xef: /* out dx,(e/r)ax */
4105 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4107 c->dst.type = OP_NONE; /* Disable writeback. */
4109 case 0xf4: /* hlt */
4110 ctxt->ops->halt(ctxt);
4112 case 0xf5: /* cmc */
4113 /* complement carry flag from eflags reg */
4114 ctxt->eflags ^= EFLG_CF;
4116 case 0xf6 ... 0xf7: /* Grp3 */
4119 case 0xf8: /* clc */
4120 ctxt->eflags &= ~EFLG_CF;
4122 case 0xf9: /* stc */
4123 ctxt->eflags |= EFLG_CF;
4125 case 0xfa: /* cli */
4126 if (emulator_bad_iopl(ctxt, ops)) {
4127 rc = emulate_gp(ctxt, 0);
4130 ctxt->eflags &= ~X86_EFLAGS_IF;
4132 case 0xfb: /* sti */
4133 if (emulator_bad_iopl(ctxt, ops)) {
4134 rc = emulate_gp(ctxt, 0);
4137 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4138 ctxt->eflags |= X86_EFLAGS_IF;
4141 case 0xfc: /* cld */
4142 ctxt->eflags &= ~EFLG_DF;
4144 case 0xfd: /* std */
4145 ctxt->eflags |= EFLG_DF;
4147 case 0xfe: /* Grp4 */
4148 rc = em_grp45(ctxt);
4150 case 0xff: /* Grp5 */
4151 rc = em_grp45(ctxt);
4154 goto cannot_emulate;
4157 if (rc != X86EMUL_CONTINUE)
4161 rc = writeback(ctxt);
4162 if (rc != X86EMUL_CONTINUE)
4166 * restore dst type in case the decoding will be reused
4167 * (happens for string instruction )
4169 c->dst.type = saved_dst_type;
4171 if ((c->d & SrcMask) == SrcSI)
4172 string_addr_inc(ctxt, seg_override(ctxt, c),
4173 VCPU_REGS_RSI, &c->src);
4175 if ((c->d & DstMask) == DstDI)
4176 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4179 if (c->rep_prefix && (c->d & String)) {
4180 struct read_cache *r = &ctxt->decode.io_read;
4181 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4183 if (!string_insn_completed(ctxt)) {
4185 * Re-enter guest when pio read ahead buffer is empty
4186 * or, if it is not used, after each 1024 iteration.
4188 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4189 (r->end == 0 || r->end != r->pos)) {
4191 * Reset read cache. Usually happens before
4192 * decode, but since instruction is restarted
4193 * we have to do it here.
4195 ctxt->decode.mem_read.end = 0;
4196 return EMULATION_RESTART;
4198 goto done; /* skip rip writeback */
4205 if (rc == X86EMUL_PROPAGATE_FAULT)
4206 ctxt->have_exception = true;
4207 if (rc == X86EMUL_INTERCEPTED)
4208 return EMULATION_INTERCEPTED;
4210 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4214 case 0x05: /* syscall */
4215 rc = emulate_syscall(ctxt, ops);
4220 case 0x09: /* wbinvd */
4221 (ctxt->ops->wbinvd)(ctxt);
4223 case 0x08: /* invd */
4224 case 0x0d: /* GrpP (prefetch) */
4225 case 0x18: /* Grp16 (prefetch/nop) */
4227 case 0x20: /* mov cr, reg */
4228 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4230 case 0x21: /* mov from dr to reg */
4231 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4233 case 0x22: /* mov reg, cr */
4234 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4235 emulate_gp(ctxt, 0);
4236 rc = X86EMUL_PROPAGATE_FAULT;
4239 c->dst.type = OP_NONE;
4241 case 0x23: /* mov from reg to dr */
4242 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4243 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4244 ~0ULL : ~0U)) < 0) {
4245 /* #UD condition is already handled by the code above */
4246 emulate_gp(ctxt, 0);
4247 rc = X86EMUL_PROPAGATE_FAULT;
4251 c->dst.type = OP_NONE; /* no writeback */
4255 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4256 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4257 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4258 emulate_gp(ctxt, 0);
4259 rc = X86EMUL_PROPAGATE_FAULT;
4262 rc = X86EMUL_CONTINUE;
4266 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4267 emulate_gp(ctxt, 0);
4268 rc = X86EMUL_PROPAGATE_FAULT;
4271 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4272 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4274 rc = X86EMUL_CONTINUE;
4276 case 0x34: /* sysenter */
4277 rc = emulate_sysenter(ctxt, ops);
4279 case 0x35: /* sysexit */
4280 rc = emulate_sysexit(ctxt, ops);
4282 case 0x40 ... 0x4f: /* cmov */
4283 c->dst.val = c->dst.orig_val = c->src.val;
4284 if (!test_cc(c->b, ctxt->eflags))
4285 c->dst.type = OP_NONE; /* no writeback */
4287 case 0x80 ... 0x8f: /* jnz rel, etc*/
4288 if (test_cc(c->b, ctxt->eflags))
4289 jmp_rel(c, c->src.val);
4291 case 0x90 ... 0x9f: /* setcc r/m8 */
4292 c->dst.val = test_cc(c->b, ctxt->eflags);
4294 case 0xa0: /* push fs */
4295 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
4297 case 0xa1: /* pop fs */
4298 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
4302 c->dst.type = OP_NONE;
4303 /* only subword offset */
4304 c->src.val &= (c->dst.bytes << 3) - 1;
4305 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4307 case 0xa4: /* shld imm8, r, r/m */
4308 case 0xa5: /* shld cl, r, r/m */
4309 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4311 case 0xa8: /* push gs */
4312 rc = emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
4314 case 0xa9: /* pop gs */
4315 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
4319 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4321 case 0xac: /* shrd imm8, r, r/m */
4322 case 0xad: /* shrd cl, r, r/m */
4323 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4325 case 0xae: /* clflush */
4327 case 0xb0 ... 0xb1: /* cmpxchg */
4329 * Save real source value, then compare EAX against
4332 c->src.orig_val = c->src.val;
4333 c->src.val = c->regs[VCPU_REGS_RAX];
4334 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4335 if (ctxt->eflags & EFLG_ZF) {
4336 /* Success: write back to memory. */
4337 c->dst.val = c->src.orig_val;
4339 /* Failure: write the value we saw to EAX. */
4340 c->dst.type = OP_REG;
4341 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4344 case 0xb2: /* lss */
4345 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_SS);
4349 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4351 case 0xb4: /* lfs */
4352 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_FS);
4354 case 0xb5: /* lgs */
4355 rc = emulate_load_segment(ctxt, ops, VCPU_SREG_GS);
4357 case 0xb6 ... 0xb7: /* movzx */
4358 c->dst.bytes = c->op_bytes;
4359 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4362 case 0xba: /* Grp8 */
4363 switch (c->modrm_reg & 3) {
4376 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4378 case 0xbc: { /* bsf */
4380 __asm__ ("bsf %2, %0; setz %1"
4381 : "=r"(c->dst.val), "=q"(zf)
4383 ctxt->eflags &= ~X86_EFLAGS_ZF;
4385 ctxt->eflags |= X86_EFLAGS_ZF;
4386 c->dst.type = OP_NONE; /* Disable writeback. */
4390 case 0xbd: { /* bsr */
4392 __asm__ ("bsr %2, %0; setz %1"
4393 : "=r"(c->dst.val), "=q"(zf)
4395 ctxt->eflags &= ~X86_EFLAGS_ZF;
4397 ctxt->eflags |= X86_EFLAGS_ZF;
4398 c->dst.type = OP_NONE; /* Disable writeback. */
4402 case 0xbe ... 0xbf: /* movsx */
4403 c->dst.bytes = c->op_bytes;
4404 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4407 case 0xc0 ... 0xc1: /* xadd */
4408 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4409 /* Write back the register source. */
4410 c->src.val = c->dst.orig_val;
4411 write_register_operand(&c->src);
4413 case 0xc3: /* movnti */
4414 c->dst.bytes = c->op_bytes;
4415 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4418 case 0xc7: /* Grp9 (cmpxchg8b) */
4422 goto cannot_emulate;
4425 if (rc != X86EMUL_CONTINUE)
4431 return EMULATION_FAILED;