1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
32 * Opcode effective-address decode tables.
33 * Note that we only emulate instructions that have at least one memory
34 * operand (excluding implicit stack references). We assume that stack
35 * references and instruction fetches will never occur in special memory
36 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
40 /* Operand sizes: 8-bit operands or specified/overridden size. */
41 #define ByteOp (1<<0) /* 8-bit operands. */
42 /* Destination operand type. */
43 #define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */
44 #define DstReg (2<<1) /* Register operand. */
45 #define DstMem (3<<1) /* Memory operand. */
46 #define DstAcc (4<<1) /* Destination Accumulator */
47 #define DstDI (5<<1) /* Destination is in ES:(E)DI */
48 #define DstMem64 (6<<1) /* 64bit memory operand */
49 #define DstImmUByte (7<<1) /* 8-bit unsigned immediate operand */
50 #define DstDX (8<<1) /* Destination is in DX register */
51 #define DstMask (0xf<<1)
52 /* Source operand type. */
53 #define SrcNone (0<<5) /* No source operand. */
54 #define SrcReg (1<<5) /* Register operand. */
55 #define SrcMem (2<<5) /* Memory operand. */
56 #define SrcMem16 (3<<5) /* Memory operand (16-bit). */
57 #define SrcMem32 (4<<5) /* Memory operand (32-bit). */
58 #define SrcImm (5<<5) /* Immediate operand. */
59 #define SrcImmByte (6<<5) /* 8-bit sign-extended immediate operand. */
60 #define SrcOne (7<<5) /* Implied '1' */
61 #define SrcImmUByte (8<<5) /* 8-bit unsigned immediate operand. */
62 #define SrcImmU (9<<5) /* Immediate operand, unsigned */
63 #define SrcSI (0xa<<5) /* Source is in the DS:RSI */
64 #define SrcImmFAddr (0xb<<5) /* Source is immediate far address */
65 #define SrcMemFAddr (0xc<<5) /* Source is far address in memory */
66 #define SrcAcc (0xd<<5) /* Source Accumulator */
67 #define SrcImmU16 (0xe<<5) /* Immediate operand, unsigned, 16 bits */
68 #define SrcDX (0xf<<5) /* Source is in DX register */
69 #define SrcMask (0xf<<5)
70 /* Generic ModRM decode. */
72 /* Destination is only written; never read. */
75 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
76 #define String (1<<13) /* String instruction (rep capable) */
77 #define Stack (1<<14) /* Stack instruction (push/pop) */
78 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
79 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
80 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
81 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
82 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
83 #define Sse (1<<18) /* SSE Vector instruction */
85 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
86 #define VendorSpecific (1<<22) /* Vendor specific instruction */
87 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
88 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
89 #define Undefined (1<<25) /* No Such Instruction */
90 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
91 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
93 /* Source 2 operand type */
94 #define Src2None (0<<29)
95 #define Src2CL (1<<29)
96 #define Src2ImmByte (2<<29)
97 #define Src2One (3<<29)
98 #define Src2Imm (4<<29)
99 #define Src2Mask (7<<29)
101 #define X2(x...) x, x
102 #define X3(x...) X2(x), x
103 #define X4(x...) X2(x), X2(x)
104 #define X5(x...) X4(x), x
105 #define X6(x...) X4(x), X2(x)
106 #define X7(x...) X4(x), X3(x)
107 #define X8(x...) X4(x), X4(x)
108 #define X16(x...) X8(x), X8(x)
114 int (*execute)(struct x86_emulate_ctxt *ctxt);
115 struct opcode *group;
116 struct group_dual *gdual;
117 struct gprefix *gprefix;
119 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
123 struct opcode mod012[8];
124 struct opcode mod3[8];
128 struct opcode pfx_no;
129 struct opcode pfx_66;
130 struct opcode pfx_f2;
131 struct opcode pfx_f3;
134 /* EFLAGS bit definitions. */
135 #define EFLG_ID (1<<21)
136 #define EFLG_VIP (1<<20)
137 #define EFLG_VIF (1<<19)
138 #define EFLG_AC (1<<18)
139 #define EFLG_VM (1<<17)
140 #define EFLG_RF (1<<16)
141 #define EFLG_IOPL (3<<12)
142 #define EFLG_NT (1<<14)
143 #define EFLG_OF (1<<11)
144 #define EFLG_DF (1<<10)
145 #define EFLG_IF (1<<9)
146 #define EFLG_TF (1<<8)
147 #define EFLG_SF (1<<7)
148 #define EFLG_ZF (1<<6)
149 #define EFLG_AF (1<<4)
150 #define EFLG_PF (1<<2)
151 #define EFLG_CF (1<<0)
153 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
154 #define EFLG_RESERVED_ONE_MASK 2
157 * Instruction emulation:
158 * Most instructions are emulated directly via a fragment of inline assembly
159 * code. This allows us to save/restore EFLAGS and thus very easily pick up
160 * any modified flags.
163 #if defined(CONFIG_X86_64)
164 #define _LO32 "k" /* force 32-bit operand */
165 #define _STK "%%rsp" /* stack pointer */
166 #elif defined(__i386__)
167 #define _LO32 "" /* force 32-bit operand */
168 #define _STK "%%esp" /* stack pointer */
172 * These EFLAGS bits are restored from saved value during emulation, and
173 * any changes are written back to the saved value after emulation.
175 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
177 /* Before executing instruction: restore necessary bits in EFLAGS. */
178 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
179 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
180 "movl %"_sav",%"_LO32 _tmp"; " \
183 "movl %"_msk",%"_LO32 _tmp"; " \
184 "andl %"_LO32 _tmp",("_STK"); " \
186 "notl %"_LO32 _tmp"; " \
187 "andl %"_LO32 _tmp",("_STK"); " \
188 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
190 "orl %"_LO32 _tmp",("_STK"); " \
194 /* After executing instruction: write-back necessary bits in EFLAGS. */
195 #define _POST_EFLAGS(_sav, _msk, _tmp) \
196 /* _sav |= EFLAGS & _msk; */ \
199 "andl %"_msk",%"_LO32 _tmp"; " \
200 "orl %"_LO32 _tmp",%"_sav"; "
208 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix, _dsttype) \
210 __asm__ __volatile__ ( \
211 _PRE_EFLAGS("0", "4", "2") \
212 _op _suffix " %"_x"3,%1; " \
213 _POST_EFLAGS("0", "4", "2") \
214 : "=m" (_eflags), "+q" (*(_dsttype*)&(_dst).val),\
216 : _y ((_src).val), "i" (EFLAGS_MASK)); \
220 /* Raw emulation: instruction has two explicit operands. */
221 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
223 unsigned long _tmp; \
225 switch ((_dst).bytes) { \
227 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w",u16);\
230 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l",u32);\
233 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q",u64)); \
238 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
240 unsigned long _tmp; \
241 switch ((_dst).bytes) { \
243 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b",u8); \
246 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
247 _wx, _wy, _lx, _ly, _qx, _qy); \
252 /* Source operand is byte-sized and may be restricted to just %cl. */
253 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
254 __emulate_2op(_op, _src, _dst, _eflags, \
255 "b", "c", "b", "c", "b", "c", "b", "c")
257 /* Source operand is byte, word, long or quad sized. */
258 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
259 __emulate_2op(_op, _src, _dst, _eflags, \
260 "b", "q", "w", "r", _LO32, "r", "", "r")
262 /* Source operand is word, long or quad sized. */
263 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
264 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
265 "w", "r", _LO32, "r", "", "r")
267 /* Instruction has three operands and one operand is stored in ECX register */
268 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
270 unsigned long _tmp; \
271 _type _clv = (_cl).val; \
272 _type _srcv = (_src).val; \
273 _type _dstv = (_dst).val; \
275 __asm__ __volatile__ ( \
276 _PRE_EFLAGS("0", "5", "2") \
277 _op _suffix " %4,%1 \n" \
278 _POST_EFLAGS("0", "5", "2") \
279 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
280 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
283 (_cl).val = (unsigned long) _clv; \
284 (_src).val = (unsigned long) _srcv; \
285 (_dst).val = (unsigned long) _dstv; \
288 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
290 switch ((_dst).bytes) { \
292 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
293 "w", unsigned short); \
296 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
297 "l", unsigned int); \
300 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
301 "q", unsigned long)); \
306 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
308 unsigned long _tmp; \
310 __asm__ __volatile__ ( \
311 _PRE_EFLAGS("0", "3", "2") \
312 _op _suffix " %1; " \
313 _POST_EFLAGS("0", "3", "2") \
314 : "=m" (_eflags), "+m" ((_dst).val), \
316 : "i" (EFLAGS_MASK)); \
319 /* Instruction has only one explicit operand (no source operand). */
320 #define emulate_1op(_op, _dst, _eflags) \
322 switch ((_dst).bytes) { \
323 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
324 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
325 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
326 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
330 #define __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags, _suffix) \
332 unsigned long _tmp; \
334 __asm__ __volatile__ ( \
335 _PRE_EFLAGS("0", "4", "1") \
336 _op _suffix " %5; " \
337 _POST_EFLAGS("0", "4", "1") \
338 : "=m" (_eflags), "=&r" (_tmp), \
339 "+a" (_rax), "+d" (_rdx) \
340 : "i" (EFLAGS_MASK), "m" ((_src).val), \
341 "a" (_rax), "d" (_rdx)); \
344 #define __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _suffix, _ex) \
346 unsigned long _tmp; \
348 __asm__ __volatile__ ( \
349 _PRE_EFLAGS("0", "5", "1") \
351 _op _suffix " %6; " \
353 _POST_EFLAGS("0", "5", "1") \
354 ".pushsection .fixup,\"ax\" \n\t" \
355 "3: movb $1, %4 \n\t" \
358 _ASM_EXTABLE(1b, 3b) \
359 : "=m" (_eflags), "=&r" (_tmp), \
360 "+a" (_rax), "+d" (_rdx), "+qm"(_ex) \
361 : "i" (EFLAGS_MASK), "m" ((_src).val), \
362 "a" (_rax), "d" (_rdx)); \
365 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
366 #define emulate_1op_rax_rdx(_op, _src, _rax, _rdx, _eflags) \
368 switch((_src).bytes) { \
370 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
374 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
378 __emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
382 ON64(__emulate_1op_rax_rdx(_op, _src, _rax, _rdx, \
388 #define emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, _eflags, _ex) \
390 switch((_src).bytes) { \
392 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
393 _eflags, "b", _ex); \
396 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
397 _eflags, "w", _ex); \
400 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
401 _eflags, "l", _ex); \
404 __emulate_1op_rax_rdx_ex(_op, _src, _rax, _rdx, \
405 _eflags, "q", _ex)); \
410 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
411 enum x86_intercept intercept,
412 enum x86_intercept_stage stage)
414 struct x86_instruction_info info = {
415 .intercept = intercept,
416 .rep_prefix = ctxt->decode.rep_prefix,
417 .modrm_mod = ctxt->decode.modrm_mod,
418 .modrm_reg = ctxt->decode.modrm_reg,
419 .modrm_rm = ctxt->decode.modrm_rm,
420 .src_val = ctxt->decode.src.val64,
421 .src_bytes = ctxt->decode.src.bytes,
422 .dst_bytes = ctxt->decode.dst.bytes,
423 .ad_bytes = ctxt->decode.ad_bytes,
424 .next_rip = ctxt->eip,
427 return ctxt->ops->intercept(ctxt, &info, stage);
430 static inline unsigned long ad_mask(struct decode_cache *c)
432 return (1UL << (c->ad_bytes << 3)) - 1;
435 /* Access/update address held in a register, based on addressing mode. */
436 static inline unsigned long
437 address_mask(struct decode_cache *c, unsigned long reg)
439 if (c->ad_bytes == sizeof(unsigned long))
442 return reg & ad_mask(c);
445 static inline unsigned long
446 register_address(struct decode_cache *c, unsigned long reg)
448 return address_mask(c, reg);
452 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
454 if (c->ad_bytes == sizeof(unsigned long))
457 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
460 static inline void jmp_rel(struct decode_cache *c, int rel)
462 register_address_increment(c, &c->eip, rel);
465 static u32 desc_limit_scaled(struct desc_struct *desc)
467 u32 limit = get_desc_limit(desc);
469 return desc->g ? (limit << 12) | 0xfff : limit;
472 static void set_seg_override(struct decode_cache *c, int seg)
474 c->has_seg_override = true;
475 c->seg_override = seg;
478 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
480 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
483 return ctxt->ops->get_cached_segment_base(ctxt, seg);
486 static unsigned seg_override(struct x86_emulate_ctxt *ctxt,
487 struct decode_cache *c)
489 if (!c->has_seg_override)
492 return c->seg_override;
495 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
496 u32 error, bool valid)
498 ctxt->exception.vector = vec;
499 ctxt->exception.error_code = error;
500 ctxt->exception.error_code_valid = valid;
501 return X86EMUL_PROPAGATE_FAULT;
504 static int emulate_db(struct x86_emulate_ctxt *ctxt)
506 return emulate_exception(ctxt, DB_VECTOR, 0, false);
509 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
511 return emulate_exception(ctxt, GP_VECTOR, err, true);
514 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
516 return emulate_exception(ctxt, SS_VECTOR, err, true);
519 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
521 return emulate_exception(ctxt, UD_VECTOR, 0, false);
524 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
526 return emulate_exception(ctxt, TS_VECTOR, err, true);
529 static int emulate_de(struct x86_emulate_ctxt *ctxt)
531 return emulate_exception(ctxt, DE_VECTOR, 0, false);
534 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
536 return emulate_exception(ctxt, NM_VECTOR, 0, false);
539 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
542 struct desc_struct desc;
544 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
548 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
553 struct desc_struct desc;
555 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
556 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
559 static int __linearize(struct x86_emulate_ctxt *ctxt,
560 struct segmented_address addr,
561 unsigned size, bool write, bool fetch,
564 struct decode_cache *c = &ctxt->decode;
565 struct desc_struct desc;
572 la = seg_base(ctxt, addr.seg) + addr.ea;
573 switch (ctxt->mode) {
574 case X86EMUL_MODE_REAL:
576 case X86EMUL_MODE_PROT64:
577 if (((signed long)la << 16) >> 16 != la)
578 return emulate_gp(ctxt, 0);
581 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
585 /* code segment or read-only data segment */
586 if (((desc.type & 8) || !(desc.type & 2)) && write)
588 /* unreadable code segment */
589 if (!fetch && (desc.type & 8) && !(desc.type & 2))
591 lim = desc_limit_scaled(&desc);
592 if ((desc.type & 8) || !(desc.type & 4)) {
593 /* expand-up segment */
594 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
597 /* exapand-down segment */
598 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
600 lim = desc.d ? 0xffffffff : 0xffff;
601 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
604 cpl = ctxt->ops->cpl(ctxt);
607 if (!(desc.type & 8)) {
611 } else if ((desc.type & 8) && !(desc.type & 4)) {
612 /* nonconforming code segment */
615 } else if ((desc.type & 8) && (desc.type & 4)) {
616 /* conforming code segment */
622 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : c->ad_bytes != 8)
625 return X86EMUL_CONTINUE;
627 if (addr.seg == VCPU_SREG_SS)
628 return emulate_ss(ctxt, addr.seg);
630 return emulate_gp(ctxt, addr.seg);
633 static int linearize(struct x86_emulate_ctxt *ctxt,
634 struct segmented_address addr,
635 unsigned size, bool write,
638 return __linearize(ctxt, addr, size, write, false, linear);
642 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
643 struct segmented_address addr,
650 rc = linearize(ctxt, addr, size, false, &linear);
651 if (rc != X86EMUL_CONTINUE)
653 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
656 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt,
657 unsigned long eip, u8 *dest)
659 struct fetch_cache *fc = &ctxt->decode.fetch;
663 if (eip == fc->end) {
664 unsigned long linear;
665 struct segmented_address addr = { .seg=VCPU_SREG_CS, .ea=eip};
666 cur_size = fc->end - fc->start;
667 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
668 rc = __linearize(ctxt, addr, size, false, true, &linear);
669 if (rc != X86EMUL_CONTINUE)
671 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
672 size, &ctxt->exception);
673 if (rc != X86EMUL_CONTINUE)
677 *dest = fc->data[eip - fc->start];
678 return X86EMUL_CONTINUE;
681 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
682 unsigned long eip, void *dest, unsigned size)
686 /* x86 instructions are limited to 15 bytes. */
687 if (eip + size - ctxt->eip > 15)
688 return X86EMUL_UNHANDLEABLE;
690 rc = do_insn_fetch_byte(ctxt, eip++, dest++);
691 if (rc != X86EMUL_CONTINUE)
694 return X86EMUL_CONTINUE;
697 /* Fetch next part of the instruction being emulated. */
698 #define insn_fetch(_type, _size, _eip) \
699 ({ unsigned long _x; \
700 rc = do_insn_fetch(ctxt, (_eip), &_x, (_size)); \
701 if (rc != X86EMUL_CONTINUE) \
707 #define insn_fetch_arr(_arr, _size, _eip) \
708 ({ rc = do_insn_fetch(ctxt, (_eip), _arr, (_size)); \
709 if (rc != X86EMUL_CONTINUE) \
715 * Given the 'reg' portion of a ModRM byte, and a register block, return a
716 * pointer into the block that addresses the relevant register.
717 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
719 static void *decode_register(u8 modrm_reg, unsigned long *regs,
724 p = ®s[modrm_reg];
725 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
726 p = (unsigned char *)®s[modrm_reg & 3] + 1;
730 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
731 struct segmented_address addr,
732 u16 *size, unsigned long *address, int op_bytes)
739 rc = segmented_read_std(ctxt, addr, size, 2);
740 if (rc != X86EMUL_CONTINUE)
743 rc = segmented_read_std(ctxt, addr, address, op_bytes);
747 static int test_cc(unsigned int condition, unsigned int flags)
751 switch ((condition & 15) >> 1) {
753 rc |= (flags & EFLG_OF);
755 case 1: /* b/c/nae */
756 rc |= (flags & EFLG_CF);
759 rc |= (flags & EFLG_ZF);
762 rc |= (flags & (EFLG_CF|EFLG_ZF));
765 rc |= (flags & EFLG_SF);
768 rc |= (flags & EFLG_PF);
771 rc |= (flags & EFLG_ZF);
774 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
778 /* Odd condition identifiers (lsb == 1) have inverted sense. */
779 return (!!rc ^ (condition & 1));
782 static void fetch_register_operand(struct operand *op)
786 op->val = *(u8 *)op->addr.reg;
789 op->val = *(u16 *)op->addr.reg;
792 op->val = *(u32 *)op->addr.reg;
795 op->val = *(u64 *)op->addr.reg;
800 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
802 ctxt->ops->get_fpu(ctxt);
804 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
805 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
806 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
807 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
808 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
809 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
810 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
811 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
813 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
814 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
815 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
816 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
817 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
818 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
819 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
820 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
824 ctxt->ops->put_fpu(ctxt);
827 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
830 ctxt->ops->get_fpu(ctxt);
832 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
833 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
834 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
835 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
836 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
837 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
838 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
839 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
841 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
842 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
843 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
844 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
845 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
846 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
847 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
848 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
852 ctxt->ops->put_fpu(ctxt);
855 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
857 struct decode_cache *c,
860 unsigned reg = c->modrm_reg;
861 int highbyte_regs = c->rex_prefix == 0;
864 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
870 read_sse_reg(ctxt, &op->vec_val, reg);
875 if ((c->d & ByteOp) && !inhibit_bytereg) {
876 op->addr.reg = decode_register(reg, c->regs, highbyte_regs);
879 op->addr.reg = decode_register(reg, c->regs, 0);
880 op->bytes = c->op_bytes;
882 fetch_register_operand(op);
883 op->orig_val = op->val;
886 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
889 struct decode_cache *c = &ctxt->decode;
891 int index_reg = 0, base_reg = 0, scale;
892 int rc = X86EMUL_CONTINUE;
896 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
897 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
898 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
901 c->modrm = insn_fetch(u8, 1, c->eip);
902 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
903 c->modrm_reg |= (c->modrm & 0x38) >> 3;
904 c->modrm_rm |= (c->modrm & 0x07);
905 c->modrm_seg = VCPU_SREG_DS;
907 if (c->modrm_mod == 3) {
909 op->bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
910 op->addr.reg = decode_register(c->modrm_rm,
911 c->regs, c->d & ByteOp);
915 op->addr.xmm = c->modrm_rm;
916 read_sse_reg(ctxt, &op->vec_val, c->modrm_rm);
919 fetch_register_operand(op);
925 if (c->ad_bytes == 2) {
926 unsigned bx = c->regs[VCPU_REGS_RBX];
927 unsigned bp = c->regs[VCPU_REGS_RBP];
928 unsigned si = c->regs[VCPU_REGS_RSI];
929 unsigned di = c->regs[VCPU_REGS_RDI];
931 /* 16-bit ModR/M decode. */
932 switch (c->modrm_mod) {
934 if (c->modrm_rm == 6)
935 modrm_ea += insn_fetch(u16, 2, c->eip);
938 modrm_ea += insn_fetch(s8, 1, c->eip);
941 modrm_ea += insn_fetch(u16, 2, c->eip);
944 switch (c->modrm_rm) {
964 if (c->modrm_mod != 0)
971 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
972 (c->modrm_rm == 6 && c->modrm_mod != 0))
973 c->modrm_seg = VCPU_SREG_SS;
974 modrm_ea = (u16)modrm_ea;
976 /* 32/64-bit ModR/M decode. */
977 if ((c->modrm_rm & 7) == 4) {
978 sib = insn_fetch(u8, 1, c->eip);
979 index_reg |= (sib >> 3) & 7;
983 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
984 modrm_ea += insn_fetch(s32, 4, c->eip);
986 modrm_ea += c->regs[base_reg];
988 modrm_ea += c->regs[index_reg] << scale;
989 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
990 if (ctxt->mode == X86EMUL_MODE_PROT64)
993 modrm_ea += c->regs[c->modrm_rm];
994 switch (c->modrm_mod) {
996 if (c->modrm_rm == 5)
997 modrm_ea += insn_fetch(s32, 4, c->eip);
1000 modrm_ea += insn_fetch(s8, 1, c->eip);
1003 modrm_ea += insn_fetch(s32, 4, c->eip);
1007 op->addr.mem.ea = modrm_ea;
1012 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1015 struct decode_cache *c = &ctxt->decode;
1016 int rc = X86EMUL_CONTINUE;
1019 switch (c->ad_bytes) {
1021 op->addr.mem.ea = insn_fetch(u16, 2, c->eip);
1024 op->addr.mem.ea = insn_fetch(u32, 4, c->eip);
1027 op->addr.mem.ea = insn_fetch(u64, 8, c->eip);
1034 static void fetch_bit_operand(struct decode_cache *c)
1038 if (c->dst.type == OP_MEM && c->src.type == OP_REG) {
1039 mask = ~(c->dst.bytes * 8 - 1);
1041 if (c->src.bytes == 2)
1042 sv = (s16)c->src.val & (s16)mask;
1043 else if (c->src.bytes == 4)
1044 sv = (s32)c->src.val & (s32)mask;
1046 c->dst.addr.mem.ea += (sv >> 3);
1049 /* only subword offset */
1050 c->src.val &= (c->dst.bytes << 3) - 1;
1053 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1054 unsigned long addr, void *dest, unsigned size)
1057 struct read_cache *mc = &ctxt->decode.mem_read;
1060 int n = min(size, 8u);
1062 if (mc->pos < mc->end)
1065 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
1067 if (rc != X86EMUL_CONTINUE)
1072 memcpy(dest, mc->data + mc->pos, n);
1077 return X86EMUL_CONTINUE;
1080 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1081 struct segmented_address addr,
1088 rc = linearize(ctxt, addr, size, false, &linear);
1089 if (rc != X86EMUL_CONTINUE)
1091 return read_emulated(ctxt, linear, data, size);
1094 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1095 struct segmented_address addr,
1102 rc = linearize(ctxt, addr, size, true, &linear);
1103 if (rc != X86EMUL_CONTINUE)
1105 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1109 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1110 struct segmented_address addr,
1111 const void *orig_data, const void *data,
1117 rc = linearize(ctxt, addr, size, true, &linear);
1118 if (rc != X86EMUL_CONTINUE)
1120 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1121 size, &ctxt->exception);
1124 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1125 unsigned int size, unsigned short port,
1128 struct read_cache *rc = &ctxt->decode.io_read;
1130 if (rc->pos == rc->end) { /* refill pio read ahead */
1131 struct decode_cache *c = &ctxt->decode;
1132 unsigned int in_page, n;
1133 unsigned int count = c->rep_prefix ?
1134 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1135 in_page = (ctxt->eflags & EFLG_DF) ?
1136 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1137 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1138 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1142 rc->pos = rc->end = 0;
1143 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1148 memcpy(dest, rc->data + rc->pos, size);
1153 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1154 u16 selector, struct desc_ptr *dt)
1156 struct x86_emulate_ops *ops = ctxt->ops;
1158 if (selector & 1 << 2) {
1159 struct desc_struct desc;
1162 memset (dt, 0, sizeof *dt);
1163 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1166 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1167 dt->address = get_desc_base(&desc);
1169 ops->get_gdt(ctxt, dt);
1172 /* allowed just for 8 bytes segments */
1173 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1174 u16 selector, struct desc_struct *desc)
1177 u16 index = selector >> 3;
1180 get_descriptor_table_ptr(ctxt, selector, &dt);
1182 if (dt.size < index * 8 + 7)
1183 return emulate_gp(ctxt, selector & 0xfffc);
1185 addr = dt.address + index * 8;
1186 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1190 /* allowed just for 8 bytes segments */
1191 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1192 u16 selector, struct desc_struct *desc)
1195 u16 index = selector >> 3;
1198 get_descriptor_table_ptr(ctxt, selector, &dt);
1200 if (dt.size < index * 8 + 7)
1201 return emulate_gp(ctxt, selector & 0xfffc);
1203 addr = dt.address + index * 8;
1204 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1208 /* Does not support long mode */
1209 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1210 u16 selector, int seg)
1212 struct desc_struct seg_desc;
1214 unsigned err_vec = GP_VECTOR;
1216 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1219 memset(&seg_desc, 0, sizeof seg_desc);
1221 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1222 || ctxt->mode == X86EMUL_MODE_REAL) {
1223 /* set real mode segment descriptor */
1224 set_desc_base(&seg_desc, selector << 4);
1225 set_desc_limit(&seg_desc, 0xffff);
1232 /* NULL selector is not valid for TR, CS and SS */
1233 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1237 /* TR should be in GDT only */
1238 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1241 if (null_selector) /* for NULL selector skip all following checks */
1244 ret = read_segment_descriptor(ctxt, selector, &seg_desc);
1245 if (ret != X86EMUL_CONTINUE)
1248 err_code = selector & 0xfffc;
1249 err_vec = GP_VECTOR;
1251 /* can't load system descriptor into segment selecor */
1252 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1256 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1262 cpl = ctxt->ops->cpl(ctxt);
1267 * segment is not a writable data segment or segment
1268 * selector's RPL != CPL or segment selector's RPL != CPL
1270 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1274 if (!(seg_desc.type & 8))
1277 if (seg_desc.type & 4) {
1283 if (rpl > cpl || dpl != cpl)
1286 /* CS(RPL) <- CPL */
1287 selector = (selector & 0xfffc) | cpl;
1290 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1293 case VCPU_SREG_LDTR:
1294 if (seg_desc.s || seg_desc.type != 2)
1297 default: /* DS, ES, FS, or GS */
1299 * segment is not a data or readable code segment or
1300 * ((segment is a data or nonconforming code segment)
1301 * and (both RPL and CPL > DPL))
1303 if ((seg_desc.type & 0xa) == 0x8 ||
1304 (((seg_desc.type & 0xc) != 0xc) &&
1305 (rpl > dpl && cpl > dpl)))
1311 /* mark segment as accessed */
1313 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1314 if (ret != X86EMUL_CONTINUE)
1318 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1319 return X86EMUL_CONTINUE;
1321 emulate_exception(ctxt, err_vec, err_code, true);
1322 return X86EMUL_PROPAGATE_FAULT;
1325 static void write_register_operand(struct operand *op)
1327 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1328 switch (op->bytes) {
1330 *(u8 *)op->addr.reg = (u8)op->val;
1333 *(u16 *)op->addr.reg = (u16)op->val;
1336 *op->addr.reg = (u32)op->val;
1337 break; /* 64b: zero-extend */
1339 *op->addr.reg = op->val;
1344 static int writeback(struct x86_emulate_ctxt *ctxt)
1347 struct decode_cache *c = &ctxt->decode;
1349 switch (c->dst.type) {
1351 write_register_operand(&c->dst);
1355 rc = segmented_cmpxchg(ctxt,
1361 rc = segmented_write(ctxt,
1365 if (rc != X86EMUL_CONTINUE)
1369 write_sse_reg(ctxt, &c->dst.vec_val, c->dst.addr.xmm);
1377 return X86EMUL_CONTINUE;
1380 static int em_push(struct x86_emulate_ctxt *ctxt)
1382 struct decode_cache *c = &ctxt->decode;
1383 struct segmented_address addr;
1385 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1386 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1387 addr.seg = VCPU_SREG_SS;
1389 /* Disable writeback. */
1390 c->dst.type = OP_NONE;
1391 return segmented_write(ctxt, addr, &c->src.val, c->op_bytes);
1394 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1395 void *dest, int len)
1397 struct decode_cache *c = &ctxt->decode;
1399 struct segmented_address addr;
1401 addr.ea = register_address(c, c->regs[VCPU_REGS_RSP]);
1402 addr.seg = VCPU_SREG_SS;
1403 rc = segmented_read(ctxt, addr, dest, len);
1404 if (rc != X86EMUL_CONTINUE)
1407 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1411 static int em_pop(struct x86_emulate_ctxt *ctxt)
1413 struct decode_cache *c = &ctxt->decode;
1415 return emulate_pop(ctxt, &c->dst.val, c->op_bytes);
1418 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1419 void *dest, int len)
1422 unsigned long val, change_mask;
1423 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1424 int cpl = ctxt->ops->cpl(ctxt);
1426 rc = emulate_pop(ctxt, &val, len);
1427 if (rc != X86EMUL_CONTINUE)
1430 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1431 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1433 switch(ctxt->mode) {
1434 case X86EMUL_MODE_PROT64:
1435 case X86EMUL_MODE_PROT32:
1436 case X86EMUL_MODE_PROT16:
1438 change_mask |= EFLG_IOPL;
1440 change_mask |= EFLG_IF;
1442 case X86EMUL_MODE_VM86:
1444 return emulate_gp(ctxt, 0);
1445 change_mask |= EFLG_IF;
1447 default: /* real mode */
1448 change_mask |= (EFLG_IOPL | EFLG_IF);
1452 *(unsigned long *)dest =
1453 (ctxt->eflags & ~change_mask) | (val & change_mask);
1458 static int em_popf(struct x86_emulate_ctxt *ctxt)
1460 struct decode_cache *c = &ctxt->decode;
1462 c->dst.type = OP_REG;
1463 c->dst.addr.reg = &ctxt->eflags;
1464 c->dst.bytes = c->op_bytes;
1465 return emulate_popf(ctxt, &c->dst.val, c->op_bytes);
1468 static int emulate_push_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1470 struct decode_cache *c = &ctxt->decode;
1472 c->src.val = get_segment_selector(ctxt, seg);
1474 return em_push(ctxt);
1477 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt, int seg)
1479 struct decode_cache *c = &ctxt->decode;
1480 unsigned long selector;
1483 rc = emulate_pop(ctxt, &selector, c->op_bytes);
1484 if (rc != X86EMUL_CONTINUE)
1487 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1491 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1493 struct decode_cache *c = &ctxt->decode;
1494 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1495 int rc = X86EMUL_CONTINUE;
1496 int reg = VCPU_REGS_RAX;
1498 while (reg <= VCPU_REGS_RDI) {
1499 (reg == VCPU_REGS_RSP) ?
1500 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1503 if (rc != X86EMUL_CONTINUE)
1512 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1514 struct decode_cache *c = &ctxt->decode;
1516 c->src.val = (unsigned long)ctxt->eflags;
1517 return em_push(ctxt);
1520 static int em_popa(struct x86_emulate_ctxt *ctxt)
1522 struct decode_cache *c = &ctxt->decode;
1523 int rc = X86EMUL_CONTINUE;
1524 int reg = VCPU_REGS_RDI;
1526 while (reg >= VCPU_REGS_RAX) {
1527 if (reg == VCPU_REGS_RSP) {
1528 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1533 rc = emulate_pop(ctxt, &c->regs[reg], c->op_bytes);
1534 if (rc != X86EMUL_CONTINUE)
1541 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1543 struct decode_cache *c = &ctxt->decode;
1544 struct x86_emulate_ops *ops = ctxt->ops;
1551 /* TODO: Add limit checks */
1552 c->src.val = ctxt->eflags;
1554 if (rc != X86EMUL_CONTINUE)
1557 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1559 c->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1561 if (rc != X86EMUL_CONTINUE)
1564 c->src.val = c->eip;
1566 if (rc != X86EMUL_CONTINUE)
1569 ops->get_idt(ctxt, &dt);
1571 eip_addr = dt.address + (irq << 2);
1572 cs_addr = dt.address + (irq << 2) + 2;
1574 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1575 if (rc != X86EMUL_CONTINUE)
1578 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1579 if (rc != X86EMUL_CONTINUE)
1582 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1583 if (rc != X86EMUL_CONTINUE)
1591 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1593 switch(ctxt->mode) {
1594 case X86EMUL_MODE_REAL:
1595 return emulate_int_real(ctxt, irq);
1596 case X86EMUL_MODE_VM86:
1597 case X86EMUL_MODE_PROT16:
1598 case X86EMUL_MODE_PROT32:
1599 case X86EMUL_MODE_PROT64:
1601 /* Protected mode interrupts unimplemented yet */
1602 return X86EMUL_UNHANDLEABLE;
1606 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1608 struct decode_cache *c = &ctxt->decode;
1609 int rc = X86EMUL_CONTINUE;
1610 unsigned long temp_eip = 0;
1611 unsigned long temp_eflags = 0;
1612 unsigned long cs = 0;
1613 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1614 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1615 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1616 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1618 /* TODO: Add stack limit check */
1620 rc = emulate_pop(ctxt, &temp_eip, c->op_bytes);
1622 if (rc != X86EMUL_CONTINUE)
1625 if (temp_eip & ~0xffff)
1626 return emulate_gp(ctxt, 0);
1628 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1630 if (rc != X86EMUL_CONTINUE)
1633 rc = emulate_pop(ctxt, &temp_eflags, c->op_bytes);
1635 if (rc != X86EMUL_CONTINUE)
1638 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1640 if (rc != X86EMUL_CONTINUE)
1646 if (c->op_bytes == 4)
1647 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1648 else if (c->op_bytes == 2) {
1649 ctxt->eflags &= ~0xffff;
1650 ctxt->eflags |= temp_eflags;
1653 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1654 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1659 static int em_iret(struct x86_emulate_ctxt *ctxt)
1661 switch(ctxt->mode) {
1662 case X86EMUL_MODE_REAL:
1663 return emulate_iret_real(ctxt);
1664 case X86EMUL_MODE_VM86:
1665 case X86EMUL_MODE_PROT16:
1666 case X86EMUL_MODE_PROT32:
1667 case X86EMUL_MODE_PROT64:
1669 /* iret from protected mode unimplemented yet */
1670 return X86EMUL_UNHANDLEABLE;
1674 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1676 struct decode_cache *c = &ctxt->decode;
1680 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1682 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1683 if (rc != X86EMUL_CONTINUE)
1687 memcpy(&c->eip, c->src.valptr, c->op_bytes);
1688 return X86EMUL_CONTINUE;
1691 static int em_grp1a(struct x86_emulate_ctxt *ctxt)
1693 struct decode_cache *c = &ctxt->decode;
1695 return emulate_pop(ctxt, &c->dst.val, c->dst.bytes);
1698 static int em_grp2(struct x86_emulate_ctxt *ctxt)
1700 struct decode_cache *c = &ctxt->decode;
1701 switch (c->modrm_reg) {
1703 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1706 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1709 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1712 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1714 case 4: /* sal/shl */
1715 case 6: /* sal/shl */
1716 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1719 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1722 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1725 return X86EMUL_CONTINUE;
1728 static int em_grp3(struct x86_emulate_ctxt *ctxt)
1730 struct decode_cache *c = &ctxt->decode;
1731 unsigned long *rax = &c->regs[VCPU_REGS_RAX];
1732 unsigned long *rdx = &c->regs[VCPU_REGS_RDX];
1735 switch (c->modrm_reg) {
1736 case 0 ... 1: /* test */
1737 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1740 c->dst.val = ~c->dst.val;
1743 emulate_1op("neg", c->dst, ctxt->eflags);
1746 emulate_1op_rax_rdx("mul", c->src, *rax, *rdx, ctxt->eflags);
1749 emulate_1op_rax_rdx("imul", c->src, *rax, *rdx, ctxt->eflags);
1752 emulate_1op_rax_rdx_ex("div", c->src, *rax, *rdx,
1756 emulate_1op_rax_rdx_ex("idiv", c->src, *rax, *rdx,
1760 return X86EMUL_UNHANDLEABLE;
1763 return emulate_de(ctxt);
1764 return X86EMUL_CONTINUE;
1767 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1769 struct decode_cache *c = &ctxt->decode;
1770 int rc = X86EMUL_CONTINUE;
1772 switch (c->modrm_reg) {
1774 emulate_1op("inc", c->dst, ctxt->eflags);
1777 emulate_1op("dec", c->dst, ctxt->eflags);
1779 case 2: /* call near abs */ {
1782 c->eip = c->src.val;
1783 c->src.val = old_eip;
1787 case 4: /* jmp abs */
1788 c->eip = c->src.val;
1790 case 5: /* jmp far */
1791 rc = em_jmp_far(ctxt);
1800 static int em_grp9(struct x86_emulate_ctxt *ctxt)
1802 struct decode_cache *c = &ctxt->decode;
1803 u64 old = c->dst.orig_val64;
1805 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1806 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1807 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1808 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1809 ctxt->eflags &= ~EFLG_ZF;
1811 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1812 (u32) c->regs[VCPU_REGS_RBX];
1814 ctxt->eflags |= EFLG_ZF;
1816 return X86EMUL_CONTINUE;
1819 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
1821 struct decode_cache *c = &ctxt->decode;
1825 rc = emulate_pop(ctxt, &c->eip, c->op_bytes);
1826 if (rc != X86EMUL_CONTINUE)
1828 if (c->op_bytes == 4)
1829 c->eip = (u32)c->eip;
1830 rc = emulate_pop(ctxt, &cs, c->op_bytes);
1831 if (rc != X86EMUL_CONTINUE)
1833 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1837 static int emulate_load_segment(struct x86_emulate_ctxt *ctxt, int seg)
1839 struct decode_cache *c = &ctxt->decode;
1843 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
1845 rc = load_segment_descriptor(ctxt, sel, seg);
1846 if (rc != X86EMUL_CONTINUE)
1849 c->dst.val = c->src.val;
1854 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1855 struct desc_struct *cs, struct desc_struct *ss)
1859 memset(cs, 0, sizeof(struct desc_struct));
1860 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
1861 memset(ss, 0, sizeof(struct desc_struct));
1863 cs->l = 0; /* will be adjusted later */
1864 set_desc_base(cs, 0); /* flat segment */
1865 cs->g = 1; /* 4kb granularity */
1866 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1867 cs->type = 0x0b; /* Read, Execute, Accessed */
1869 cs->dpl = 0; /* will be adjusted later */
1873 set_desc_base(ss, 0); /* flat segment */
1874 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1875 ss->g = 1; /* 4kb granularity */
1877 ss->type = 0x03; /* Read/Write, Accessed */
1878 ss->d = 1; /* 32bit stack segment */
1883 static int em_syscall(struct x86_emulate_ctxt *ctxt)
1885 struct decode_cache *c = &ctxt->decode;
1886 struct x86_emulate_ops *ops = ctxt->ops;
1887 struct desc_struct cs, ss;
1892 /* syscall is not available in real mode */
1893 if (ctxt->mode == X86EMUL_MODE_REAL ||
1894 ctxt->mode == X86EMUL_MODE_VM86)
1895 return emulate_ud(ctxt);
1897 ops->get_msr(ctxt, MSR_EFER, &efer);
1898 setup_syscalls_segments(ctxt, &cs, &ss);
1900 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1902 cs_sel = (u16)(msr_data & 0xfffc);
1903 ss_sel = (u16)(msr_data + 8);
1905 if (efer & EFER_LMA) {
1909 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1910 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1912 c->regs[VCPU_REGS_RCX] = c->eip;
1913 if (efer & EFER_LMA) {
1914 #ifdef CONFIG_X86_64
1915 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
1918 ctxt->mode == X86EMUL_MODE_PROT64 ?
1919 MSR_LSTAR : MSR_CSTAR, &msr_data);
1922 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
1923 ctxt->eflags &= ~(msr_data | EFLG_RF);
1927 ops->get_msr(ctxt, MSR_STAR, &msr_data);
1928 c->eip = (u32)msr_data;
1930 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1933 return X86EMUL_CONTINUE;
1936 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
1938 struct decode_cache *c = &ctxt->decode;
1939 struct x86_emulate_ops *ops = ctxt->ops;
1940 struct desc_struct cs, ss;
1945 ops->get_msr(ctxt, MSR_EFER, &efer);
1946 /* inject #GP if in real mode */
1947 if (ctxt->mode == X86EMUL_MODE_REAL)
1948 return emulate_gp(ctxt, 0);
1950 /* XXX sysenter/sysexit have not been tested in 64bit mode.
1951 * Therefore, we inject an #UD.
1953 if (ctxt->mode == X86EMUL_MODE_PROT64)
1954 return emulate_ud(ctxt);
1956 setup_syscalls_segments(ctxt, &cs, &ss);
1958 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
1959 switch (ctxt->mode) {
1960 case X86EMUL_MODE_PROT32:
1961 if ((msr_data & 0xfffc) == 0x0)
1962 return emulate_gp(ctxt, 0);
1964 case X86EMUL_MODE_PROT64:
1965 if (msr_data == 0x0)
1966 return emulate_gp(ctxt, 0);
1970 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
1971 cs_sel = (u16)msr_data;
1972 cs_sel &= ~SELECTOR_RPL_MASK;
1973 ss_sel = cs_sel + 8;
1974 ss_sel &= ~SELECTOR_RPL_MASK;
1975 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
1980 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
1981 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
1983 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
1986 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
1987 c->regs[VCPU_REGS_RSP] = msr_data;
1989 return X86EMUL_CONTINUE;
1992 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
1994 struct decode_cache *c = &ctxt->decode;
1995 struct x86_emulate_ops *ops = ctxt->ops;
1996 struct desc_struct cs, ss;
1999 u16 cs_sel = 0, ss_sel = 0;
2001 /* inject #GP if in real mode or Virtual 8086 mode */
2002 if (ctxt->mode == X86EMUL_MODE_REAL ||
2003 ctxt->mode == X86EMUL_MODE_VM86)
2004 return emulate_gp(ctxt, 0);
2006 setup_syscalls_segments(ctxt, &cs, &ss);
2008 if ((c->rex_prefix & 0x8) != 0x0)
2009 usermode = X86EMUL_MODE_PROT64;
2011 usermode = X86EMUL_MODE_PROT32;
2015 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2017 case X86EMUL_MODE_PROT32:
2018 cs_sel = (u16)(msr_data + 16);
2019 if ((msr_data & 0xfffc) == 0x0)
2020 return emulate_gp(ctxt, 0);
2021 ss_sel = (u16)(msr_data + 24);
2023 case X86EMUL_MODE_PROT64:
2024 cs_sel = (u16)(msr_data + 32);
2025 if (msr_data == 0x0)
2026 return emulate_gp(ctxt, 0);
2027 ss_sel = cs_sel + 8;
2032 cs_sel |= SELECTOR_RPL_MASK;
2033 ss_sel |= SELECTOR_RPL_MASK;
2035 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2036 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2038 c->eip = c->regs[VCPU_REGS_RDX];
2039 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2041 return X86EMUL_CONTINUE;
2044 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2047 if (ctxt->mode == X86EMUL_MODE_REAL)
2049 if (ctxt->mode == X86EMUL_MODE_VM86)
2051 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2052 return ctxt->ops->cpl(ctxt) > iopl;
2055 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2058 struct x86_emulate_ops *ops = ctxt->ops;
2059 struct desc_struct tr_seg;
2062 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2063 unsigned mask = (1 << len) - 1;
2066 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2069 if (desc_limit_scaled(&tr_seg) < 103)
2071 base = get_desc_base(&tr_seg);
2072 #ifdef CONFIG_X86_64
2073 base |= ((u64)base3) << 32;
2075 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2076 if (r != X86EMUL_CONTINUE)
2078 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2080 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2081 if (r != X86EMUL_CONTINUE)
2083 if ((perm >> bit_idx) & mask)
2088 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2094 if (emulator_bad_iopl(ctxt))
2095 if (!emulator_io_port_access_allowed(ctxt, port, len))
2098 ctxt->perm_ok = true;
2103 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2104 struct tss_segment_16 *tss)
2106 struct decode_cache *c = &ctxt->decode;
2109 tss->flag = ctxt->eflags;
2110 tss->ax = c->regs[VCPU_REGS_RAX];
2111 tss->cx = c->regs[VCPU_REGS_RCX];
2112 tss->dx = c->regs[VCPU_REGS_RDX];
2113 tss->bx = c->regs[VCPU_REGS_RBX];
2114 tss->sp = c->regs[VCPU_REGS_RSP];
2115 tss->bp = c->regs[VCPU_REGS_RBP];
2116 tss->si = c->regs[VCPU_REGS_RSI];
2117 tss->di = c->regs[VCPU_REGS_RDI];
2119 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2120 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2121 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2122 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2123 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2126 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2127 struct tss_segment_16 *tss)
2129 struct decode_cache *c = &ctxt->decode;
2133 ctxt->eflags = tss->flag | 2;
2134 c->regs[VCPU_REGS_RAX] = tss->ax;
2135 c->regs[VCPU_REGS_RCX] = tss->cx;
2136 c->regs[VCPU_REGS_RDX] = tss->dx;
2137 c->regs[VCPU_REGS_RBX] = tss->bx;
2138 c->regs[VCPU_REGS_RSP] = tss->sp;
2139 c->regs[VCPU_REGS_RBP] = tss->bp;
2140 c->regs[VCPU_REGS_RSI] = tss->si;
2141 c->regs[VCPU_REGS_RDI] = tss->di;
2144 * SDM says that segment selectors are loaded before segment
2147 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2148 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2149 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2150 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2151 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2154 * Now load segment descriptors. If fault happenes at this stage
2155 * it is handled in a context of new task
2157 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2158 if (ret != X86EMUL_CONTINUE)
2160 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2161 if (ret != X86EMUL_CONTINUE)
2163 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2164 if (ret != X86EMUL_CONTINUE)
2166 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2167 if (ret != X86EMUL_CONTINUE)
2169 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2170 if (ret != X86EMUL_CONTINUE)
2173 return X86EMUL_CONTINUE;
2176 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2177 u16 tss_selector, u16 old_tss_sel,
2178 ulong old_tss_base, struct desc_struct *new_desc)
2180 struct x86_emulate_ops *ops = ctxt->ops;
2181 struct tss_segment_16 tss_seg;
2183 u32 new_tss_base = get_desc_base(new_desc);
2185 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2187 if (ret != X86EMUL_CONTINUE)
2188 /* FIXME: need to provide precise fault address */
2191 save_state_to_tss16(ctxt, &tss_seg);
2193 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2195 if (ret != X86EMUL_CONTINUE)
2196 /* FIXME: need to provide precise fault address */
2199 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2201 if (ret != X86EMUL_CONTINUE)
2202 /* FIXME: need to provide precise fault address */
2205 if (old_tss_sel != 0xffff) {
2206 tss_seg.prev_task_link = old_tss_sel;
2208 ret = ops->write_std(ctxt, new_tss_base,
2209 &tss_seg.prev_task_link,
2210 sizeof tss_seg.prev_task_link,
2212 if (ret != X86EMUL_CONTINUE)
2213 /* FIXME: need to provide precise fault address */
2217 return load_state_from_tss16(ctxt, &tss_seg);
2220 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2221 struct tss_segment_32 *tss)
2223 struct decode_cache *c = &ctxt->decode;
2225 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2227 tss->eflags = ctxt->eflags;
2228 tss->eax = c->regs[VCPU_REGS_RAX];
2229 tss->ecx = c->regs[VCPU_REGS_RCX];
2230 tss->edx = c->regs[VCPU_REGS_RDX];
2231 tss->ebx = c->regs[VCPU_REGS_RBX];
2232 tss->esp = c->regs[VCPU_REGS_RSP];
2233 tss->ebp = c->regs[VCPU_REGS_RBP];
2234 tss->esi = c->regs[VCPU_REGS_RSI];
2235 tss->edi = c->regs[VCPU_REGS_RDI];
2237 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2238 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2239 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2240 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2241 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2242 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2243 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2246 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2247 struct tss_segment_32 *tss)
2249 struct decode_cache *c = &ctxt->decode;
2252 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2253 return emulate_gp(ctxt, 0);
2255 ctxt->eflags = tss->eflags | 2;
2256 c->regs[VCPU_REGS_RAX] = tss->eax;
2257 c->regs[VCPU_REGS_RCX] = tss->ecx;
2258 c->regs[VCPU_REGS_RDX] = tss->edx;
2259 c->regs[VCPU_REGS_RBX] = tss->ebx;
2260 c->regs[VCPU_REGS_RSP] = tss->esp;
2261 c->regs[VCPU_REGS_RBP] = tss->ebp;
2262 c->regs[VCPU_REGS_RSI] = tss->esi;
2263 c->regs[VCPU_REGS_RDI] = tss->edi;
2266 * SDM says that segment selectors are loaded before segment
2269 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2270 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2271 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2272 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2273 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2274 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2275 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2278 * Now load segment descriptors. If fault happenes at this stage
2279 * it is handled in a context of new task
2281 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2282 if (ret != X86EMUL_CONTINUE)
2284 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2285 if (ret != X86EMUL_CONTINUE)
2287 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2288 if (ret != X86EMUL_CONTINUE)
2290 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2291 if (ret != X86EMUL_CONTINUE)
2293 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2294 if (ret != X86EMUL_CONTINUE)
2296 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2297 if (ret != X86EMUL_CONTINUE)
2299 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2300 if (ret != X86EMUL_CONTINUE)
2303 return X86EMUL_CONTINUE;
2306 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2307 u16 tss_selector, u16 old_tss_sel,
2308 ulong old_tss_base, struct desc_struct *new_desc)
2310 struct x86_emulate_ops *ops = ctxt->ops;
2311 struct tss_segment_32 tss_seg;
2313 u32 new_tss_base = get_desc_base(new_desc);
2315 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2317 if (ret != X86EMUL_CONTINUE)
2318 /* FIXME: need to provide precise fault address */
2321 save_state_to_tss32(ctxt, &tss_seg);
2323 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2325 if (ret != X86EMUL_CONTINUE)
2326 /* FIXME: need to provide precise fault address */
2329 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2331 if (ret != X86EMUL_CONTINUE)
2332 /* FIXME: need to provide precise fault address */
2335 if (old_tss_sel != 0xffff) {
2336 tss_seg.prev_task_link = old_tss_sel;
2338 ret = ops->write_std(ctxt, new_tss_base,
2339 &tss_seg.prev_task_link,
2340 sizeof tss_seg.prev_task_link,
2342 if (ret != X86EMUL_CONTINUE)
2343 /* FIXME: need to provide precise fault address */
2347 return load_state_from_tss32(ctxt, &tss_seg);
2350 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2351 u16 tss_selector, int reason,
2352 bool has_error_code, u32 error_code)
2354 struct x86_emulate_ops *ops = ctxt->ops;
2355 struct desc_struct curr_tss_desc, next_tss_desc;
2357 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2358 ulong old_tss_base =
2359 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2362 /* FIXME: old_tss_base == ~0 ? */
2364 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2365 if (ret != X86EMUL_CONTINUE)
2367 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2368 if (ret != X86EMUL_CONTINUE)
2371 /* FIXME: check that next_tss_desc is tss */
2373 if (reason != TASK_SWITCH_IRET) {
2374 if ((tss_selector & 3) > next_tss_desc.dpl ||
2375 ops->cpl(ctxt) > next_tss_desc.dpl)
2376 return emulate_gp(ctxt, 0);
2379 desc_limit = desc_limit_scaled(&next_tss_desc);
2380 if (!next_tss_desc.p ||
2381 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2382 desc_limit < 0x2b)) {
2383 emulate_ts(ctxt, tss_selector & 0xfffc);
2384 return X86EMUL_PROPAGATE_FAULT;
2387 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2388 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2389 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2392 if (reason == TASK_SWITCH_IRET)
2393 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2395 /* set back link to prev task only if NT bit is set in eflags
2396 note that old_tss_sel is not used afetr this point */
2397 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2398 old_tss_sel = 0xffff;
2400 if (next_tss_desc.type & 8)
2401 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2402 old_tss_base, &next_tss_desc);
2404 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2405 old_tss_base, &next_tss_desc);
2406 if (ret != X86EMUL_CONTINUE)
2409 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2410 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2412 if (reason != TASK_SWITCH_IRET) {
2413 next_tss_desc.type |= (1 << 1); /* set busy flag */
2414 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2417 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2418 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2420 if (has_error_code) {
2421 struct decode_cache *c = &ctxt->decode;
2423 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2425 c->src.val = (unsigned long) error_code;
2426 ret = em_push(ctxt);
2432 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2433 u16 tss_selector, int reason,
2434 bool has_error_code, u32 error_code)
2436 struct decode_cache *c = &ctxt->decode;
2440 c->dst.type = OP_NONE;
2442 rc = emulator_do_task_switch(ctxt, tss_selector, reason,
2443 has_error_code, error_code);
2445 if (rc == X86EMUL_CONTINUE)
2448 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2451 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
2452 int reg, struct operand *op)
2454 struct decode_cache *c = &ctxt->decode;
2455 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2457 register_address_increment(c, &c->regs[reg], df * op->bytes);
2458 op->addr.mem.ea = register_address(c, c->regs[reg]);
2459 op->addr.mem.seg = seg;
2462 static int em_das(struct x86_emulate_ctxt *ctxt)
2464 struct decode_cache *c = &ctxt->decode;
2466 bool af, cf, old_cf;
2468 cf = ctxt->eflags & X86_EFLAGS_CF;
2474 af = ctxt->eflags & X86_EFLAGS_AF;
2475 if ((al & 0x0f) > 9 || af) {
2477 cf = old_cf | (al >= 250);
2482 if (old_al > 0x99 || old_cf) {
2488 /* Set PF, ZF, SF */
2489 c->src.type = OP_IMM;
2492 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2493 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2495 ctxt->eflags |= X86_EFLAGS_CF;
2497 ctxt->eflags |= X86_EFLAGS_AF;
2498 return X86EMUL_CONTINUE;
2501 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2503 struct decode_cache *c = &ctxt->decode;
2508 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2511 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
2512 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2513 return X86EMUL_CONTINUE;
2516 memcpy(&c->eip, c->src.valptr, c->op_bytes);
2518 c->src.val = old_cs;
2520 if (rc != X86EMUL_CONTINUE)
2523 c->src.val = old_eip;
2524 return em_push(ctxt);
2527 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2529 struct decode_cache *c = &ctxt->decode;
2532 c->dst.type = OP_REG;
2533 c->dst.addr.reg = &c->eip;
2534 c->dst.bytes = c->op_bytes;
2535 rc = emulate_pop(ctxt, &c->dst.val, c->op_bytes);
2536 if (rc != X86EMUL_CONTINUE)
2538 register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.val);
2539 return X86EMUL_CONTINUE;
2542 static int em_add(struct x86_emulate_ctxt *ctxt)
2544 struct decode_cache *c = &ctxt->decode;
2546 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2547 return X86EMUL_CONTINUE;
2550 static int em_or(struct x86_emulate_ctxt *ctxt)
2552 struct decode_cache *c = &ctxt->decode;
2554 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2555 return X86EMUL_CONTINUE;
2558 static int em_adc(struct x86_emulate_ctxt *ctxt)
2560 struct decode_cache *c = &ctxt->decode;
2562 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2563 return X86EMUL_CONTINUE;
2566 static int em_sbb(struct x86_emulate_ctxt *ctxt)
2568 struct decode_cache *c = &ctxt->decode;
2570 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2571 return X86EMUL_CONTINUE;
2574 static int em_and(struct x86_emulate_ctxt *ctxt)
2576 struct decode_cache *c = &ctxt->decode;
2578 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2579 return X86EMUL_CONTINUE;
2582 static int em_sub(struct x86_emulate_ctxt *ctxt)
2584 struct decode_cache *c = &ctxt->decode;
2586 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2587 return X86EMUL_CONTINUE;
2590 static int em_xor(struct x86_emulate_ctxt *ctxt)
2592 struct decode_cache *c = &ctxt->decode;
2594 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2595 return X86EMUL_CONTINUE;
2598 static int em_cmp(struct x86_emulate_ctxt *ctxt)
2600 struct decode_cache *c = &ctxt->decode;
2602 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2603 /* Disable writeback. */
2604 c->dst.type = OP_NONE;
2605 return X86EMUL_CONTINUE;
2608 static int em_test(struct x86_emulate_ctxt *ctxt)
2610 struct decode_cache *c = &ctxt->decode;
2612 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2613 return X86EMUL_CONTINUE;
2616 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2618 struct decode_cache *c = &ctxt->decode;
2620 /* Write back the register source. */
2621 c->src.val = c->dst.val;
2622 write_register_operand(&c->src);
2624 /* Write back the memory destination with implicit LOCK prefix. */
2625 c->dst.val = c->src.orig_val;
2627 return X86EMUL_CONTINUE;
2630 static int em_imul(struct x86_emulate_ctxt *ctxt)
2632 struct decode_cache *c = &ctxt->decode;
2634 emulate_2op_SrcV_nobyte("imul", c->src, c->dst, ctxt->eflags);
2635 return X86EMUL_CONTINUE;
2638 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2640 struct decode_cache *c = &ctxt->decode;
2642 c->dst.val = c->src2.val;
2643 return em_imul(ctxt);
2646 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2648 struct decode_cache *c = &ctxt->decode;
2650 c->dst.type = OP_REG;
2651 c->dst.bytes = c->src.bytes;
2652 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
2653 c->dst.val = ~((c->src.val >> (c->src.bytes * 8 - 1)) - 1);
2655 return X86EMUL_CONTINUE;
2658 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2660 struct decode_cache *c = &ctxt->decode;
2663 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2664 c->regs[VCPU_REGS_RAX] = (u32)tsc;
2665 c->regs[VCPU_REGS_RDX] = tsc >> 32;
2666 return X86EMUL_CONTINUE;
2669 static int em_mov(struct x86_emulate_ctxt *ctxt)
2671 struct decode_cache *c = &ctxt->decode;
2672 c->dst.val = c->src.val;
2673 return X86EMUL_CONTINUE;
2676 static int em_movdqu(struct x86_emulate_ctxt *ctxt)
2678 struct decode_cache *c = &ctxt->decode;
2679 memcpy(&c->dst.vec_val, &c->src.vec_val, c->op_bytes);
2680 return X86EMUL_CONTINUE;
2683 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
2685 struct decode_cache *c = &ctxt->decode;
2689 rc = linearize(ctxt, c->src.addr.mem, 1, false, &linear);
2690 if (rc == X86EMUL_CONTINUE)
2691 ctxt->ops->invlpg(ctxt, linear);
2692 /* Disable writeback. */
2693 c->dst.type = OP_NONE;
2694 return X86EMUL_CONTINUE;
2697 static int em_clts(struct x86_emulate_ctxt *ctxt)
2701 cr0 = ctxt->ops->get_cr(ctxt, 0);
2703 ctxt->ops->set_cr(ctxt, 0, cr0);
2704 return X86EMUL_CONTINUE;
2707 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
2709 struct decode_cache *c = &ctxt->decode;
2712 if (c->modrm_mod != 3 || c->modrm_rm != 1)
2713 return X86EMUL_UNHANDLEABLE;
2715 rc = ctxt->ops->fix_hypercall(ctxt);
2716 if (rc != X86EMUL_CONTINUE)
2719 /* Let the processor re-execute the fixed hypercall */
2721 /* Disable writeback. */
2722 c->dst.type = OP_NONE;
2723 return X86EMUL_CONTINUE;
2726 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
2728 struct decode_cache *c = &ctxt->decode;
2729 struct desc_ptr desc_ptr;
2732 rc = read_descriptor(ctxt, c->src.addr.mem,
2733 &desc_ptr.size, &desc_ptr.address,
2735 if (rc != X86EMUL_CONTINUE)
2737 ctxt->ops->set_gdt(ctxt, &desc_ptr);
2738 /* Disable writeback. */
2739 c->dst.type = OP_NONE;
2740 return X86EMUL_CONTINUE;
2743 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
2745 struct decode_cache *c = &ctxt->decode;
2748 rc = ctxt->ops->fix_hypercall(ctxt);
2750 /* Disable writeback. */
2751 c->dst.type = OP_NONE;
2755 static int em_lidt(struct x86_emulate_ctxt *ctxt)
2757 struct decode_cache *c = &ctxt->decode;
2758 struct desc_ptr desc_ptr;
2761 rc = read_descriptor(ctxt, c->src.addr.mem,
2762 &desc_ptr.size, &desc_ptr.address,
2764 if (rc != X86EMUL_CONTINUE)
2766 ctxt->ops->set_idt(ctxt, &desc_ptr);
2767 /* Disable writeback. */
2768 c->dst.type = OP_NONE;
2769 return X86EMUL_CONTINUE;
2772 static int em_smsw(struct x86_emulate_ctxt *ctxt)
2774 struct decode_cache *c = &ctxt->decode;
2777 c->dst.val = ctxt->ops->get_cr(ctxt, 0);
2778 return X86EMUL_CONTINUE;
2781 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
2783 struct decode_cache *c = &ctxt->decode;
2784 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
2785 | (c->src.val & 0x0f));
2786 c->dst.type = OP_NONE;
2787 return X86EMUL_CONTINUE;
2790 static bool valid_cr(int nr)
2802 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
2804 struct decode_cache *c = &ctxt->decode;
2806 if (!valid_cr(c->modrm_reg))
2807 return emulate_ud(ctxt);
2809 return X86EMUL_CONTINUE;
2812 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
2814 struct decode_cache *c = &ctxt->decode;
2815 u64 new_val = c->src.val64;
2816 int cr = c->modrm_reg;
2819 static u64 cr_reserved_bits[] = {
2820 0xffffffff00000000ULL,
2821 0, 0, 0, /* CR3 checked later */
2828 return emulate_ud(ctxt);
2830 if (new_val & cr_reserved_bits[cr])
2831 return emulate_gp(ctxt, 0);
2836 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
2837 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
2838 return emulate_gp(ctxt, 0);
2840 cr4 = ctxt->ops->get_cr(ctxt, 4);
2841 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2843 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
2844 !(cr4 & X86_CR4_PAE))
2845 return emulate_gp(ctxt, 0);
2852 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2853 if (efer & EFER_LMA)
2854 rsvd = CR3_L_MODE_RESERVED_BITS;
2855 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
2856 rsvd = CR3_PAE_RESERVED_BITS;
2857 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
2858 rsvd = CR3_NONPAE_RESERVED_BITS;
2861 return emulate_gp(ctxt, 0);
2868 cr4 = ctxt->ops->get_cr(ctxt, 4);
2869 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2871 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
2872 return emulate_gp(ctxt, 0);
2878 return X86EMUL_CONTINUE;
2881 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
2885 ctxt->ops->get_dr(ctxt, 7, &dr7);
2887 /* Check if DR7.Global_Enable is set */
2888 return dr7 & (1 << 13);
2891 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
2893 struct decode_cache *c = &ctxt->decode;
2894 int dr = c->modrm_reg;
2898 return emulate_ud(ctxt);
2900 cr4 = ctxt->ops->get_cr(ctxt, 4);
2901 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
2902 return emulate_ud(ctxt);
2904 if (check_dr7_gd(ctxt))
2905 return emulate_db(ctxt);
2907 return X86EMUL_CONTINUE;
2910 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
2912 struct decode_cache *c = &ctxt->decode;
2913 u64 new_val = c->src.val64;
2914 int dr = c->modrm_reg;
2916 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
2917 return emulate_gp(ctxt, 0);
2919 return check_dr_read(ctxt);
2922 static int check_svme(struct x86_emulate_ctxt *ctxt)
2926 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
2928 if (!(efer & EFER_SVME))
2929 return emulate_ud(ctxt);
2931 return X86EMUL_CONTINUE;
2934 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
2936 u64 rax = ctxt->decode.regs[VCPU_REGS_RAX];
2938 /* Valid physical address? */
2939 if (rax & 0xffff000000000000ULL)
2940 return emulate_gp(ctxt, 0);
2942 return check_svme(ctxt);
2945 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
2947 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2949 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
2950 return emulate_ud(ctxt);
2952 return X86EMUL_CONTINUE;
2955 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
2957 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
2958 u64 rcx = ctxt->decode.regs[VCPU_REGS_RCX];
2960 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
2962 return emulate_gp(ctxt, 0);
2964 return X86EMUL_CONTINUE;
2967 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
2969 struct decode_cache *c = &ctxt->decode;
2971 c->dst.bytes = min(c->dst.bytes, 4u);
2972 if (!emulator_io_permited(ctxt, c->src.val, c->dst.bytes))
2973 return emulate_gp(ctxt, 0);
2975 return X86EMUL_CONTINUE;
2978 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
2980 struct decode_cache *c = &ctxt->decode;
2982 c->src.bytes = min(c->src.bytes, 4u);
2983 if (!emulator_io_permited(ctxt, c->dst.val, c->src.bytes))
2984 return emulate_gp(ctxt, 0);
2986 return X86EMUL_CONTINUE;
2989 #define D(_y) { .flags = (_y) }
2990 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
2991 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
2992 .check_perm = (_p) }
2994 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
2995 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
2996 #define GD(_f, _g) { .flags = ((_f) | GroupDual), .u.gdual = (_g) }
2997 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
2998 #define II(_f, _e, _i) \
2999 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3000 #define IIP(_f, _e, _i, _p) \
3001 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3002 .check_perm = (_p) }
3003 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3005 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3006 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3007 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3009 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3010 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3011 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3013 static struct opcode group7_rm1[] = {
3014 DI(SrcNone | ModRM | Priv, monitor),
3015 DI(SrcNone | ModRM | Priv, mwait),
3019 static struct opcode group7_rm3[] = {
3020 DIP(SrcNone | ModRM | Prot | Priv, vmrun, check_svme_pa),
3021 II(SrcNone | ModRM | Prot | VendorSpecific, em_vmmcall, vmmcall),
3022 DIP(SrcNone | ModRM | Prot | Priv, vmload, check_svme_pa),
3023 DIP(SrcNone | ModRM | Prot | Priv, vmsave, check_svme_pa),
3024 DIP(SrcNone | ModRM | Prot | Priv, stgi, check_svme),
3025 DIP(SrcNone | ModRM | Prot | Priv, clgi, check_svme),
3026 DIP(SrcNone | ModRM | Prot | Priv, skinit, check_svme),
3027 DIP(SrcNone | ModRM | Prot | Priv, invlpga, check_svme),
3030 static struct opcode group7_rm7[] = {
3032 DIP(SrcNone | ModRM, rdtscp, check_rdtsc),
3036 static struct opcode group1[] = {
3047 static struct opcode group1A[] = {
3048 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
3051 static struct opcode group3[] = {
3052 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
3053 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3054 X4(D(SrcMem | ModRM)),
3057 static struct opcode group4[] = {
3058 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
3062 static struct opcode group5[] = {
3063 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
3064 D(SrcMem | ModRM | Stack),
3065 I(SrcMemFAddr | ModRM | ImplicitOps | Stack, em_call_far),
3066 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
3067 D(SrcMem | ModRM | Stack), N,
3070 static struct opcode group6[] = {
3071 DI(ModRM | Prot, sldt),
3072 DI(ModRM | Prot, str),
3073 DI(ModRM | Prot | Priv, lldt),
3074 DI(ModRM | Prot | Priv, ltr),
3078 static struct group_dual group7 = { {
3079 DI(ModRM | Mov | DstMem | Priv, sgdt),
3080 DI(ModRM | Mov | DstMem | Priv, sidt),
3081 II(ModRM | SrcMem | Priv, em_lgdt, lgdt),
3082 II(ModRM | SrcMem | Priv, em_lidt, lidt),
3083 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3084 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw),
3085 II(SrcMem | ModRM | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3087 I(SrcNone | ModRM | Priv | VendorSpecific, em_vmcall),
3089 N, EXT(0, group7_rm3),
3090 II(SrcNone | ModRM | DstMem | Mov, em_smsw, smsw), N,
3091 II(SrcMem16 | ModRM | Mov | Priv, em_lmsw, lmsw), EXT(0, group7_rm7),
3094 static struct opcode group8[] = {
3096 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
3097 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
3100 static struct group_dual group9 = { {
3101 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
3103 N, N, N, N, N, N, N, N,
3106 static struct opcode group11[] = {
3107 I(DstMem | SrcImm | ModRM | Mov, em_mov), X7(D(Undefined)),
3110 static struct gprefix pfx_0f_6f_0f_7f = {
3111 N, N, N, I(Sse, em_movdqu),
3114 static struct opcode opcode_table[256] = {
3116 I6ALU(Lock, em_add),
3117 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3120 D(ImplicitOps | Stack | No64), N,
3122 I6ALU(Lock, em_adc),
3123 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3125 I6ALU(Lock, em_sbb),
3126 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
3128 I6ALU(Lock, em_and), N, N,
3130 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3132 I6ALU(Lock, em_xor), N, N,
3134 I6ALU(0, em_cmp), N, N,
3138 X8(I(SrcReg | Stack, em_push)),
3140 X8(I(DstReg | Stack, em_pop)),
3142 I(ImplicitOps | Stack | No64, em_pusha),
3143 I(ImplicitOps | Stack | No64, em_popa),
3144 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3147 I(SrcImm | Mov | Stack, em_push),
3148 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3149 I(SrcImmByte | Mov | Stack, em_push),
3150 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3151 D2bvIP(DstDI | SrcDX | Mov | String, ins, check_perm_in), /* insb, insw/insd */
3152 D2bvIP(SrcSI | DstDX | String, outs, check_perm_out), /* outsb, outsw/outsd */
3156 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
3157 G(DstMem | SrcImm | ModRM | Group, group1),
3158 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
3159 G(DstMem | SrcImmByte | ModRM | Group, group1),
3160 I2bv(DstMem | SrcReg | ModRM, em_test),
3161 I2bv(DstMem | SrcReg | ModRM | Lock, em_xchg),
3163 I2bv(DstMem | SrcReg | ModRM | Mov, em_mov),
3164 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3165 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | SrcMem | NoAccess | DstReg),
3166 D(ImplicitOps | SrcMem16 | ModRM), G(0, group1A),
3168 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3170 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3171 I(SrcImmFAddr | No64, em_call_far), N,
3172 II(ImplicitOps | Stack, em_pushf, pushf),
3173 II(ImplicitOps | Stack, em_popf, popf), N, N,
3175 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3176 I2bv(DstMem | SrcAcc | Mov | MemAbs, em_mov),
3177 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3178 I2bv(SrcSI | DstDI | String, em_cmp),
3180 I2bv(DstAcc | SrcImm, em_test),
3181 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3182 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3183 I2bv(SrcAcc | DstDI | String, em_cmp),
3185 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3187 X8(I(DstReg | SrcImm | Mov, em_mov)),
3189 D2bv(DstMem | SrcImmByte | ModRM),
3190 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3191 D(ImplicitOps | Stack),
3192 D(DstReg | SrcMemFAddr | ModRM | No64), D(DstReg | SrcMemFAddr | ModRM | No64),
3193 G(ByteOp, group11), G(0, group11),
3195 N, N, N, I(ImplicitOps | Stack, em_ret_far),
3196 D(ImplicitOps), DI(SrcImmByte, intn),
3197 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3199 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3202 N, N, N, N, N, N, N, N,
3205 D2bvIP(SrcImmUByte | DstAcc, in, check_perm_in),
3206 D2bvIP(SrcAcc | DstImmUByte, out, check_perm_out),
3208 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
3209 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3210 D2bvIP(SrcDX | DstAcc, in, check_perm_in),
3211 D2bvIP(SrcAcc | DstDX, out, check_perm_out),
3213 N, DI(ImplicitOps, icebp), N, N,
3214 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3215 G(ByteOp, group3), G(0, group3),
3217 D(ImplicitOps), D(ImplicitOps), D(ImplicitOps), D(ImplicitOps),
3218 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3221 static struct opcode twobyte_table[256] = {
3223 G(0, group6), GD(0, &group7), N, N,
3224 N, I(ImplicitOps | VendorSpecific, em_syscall),
3225 II(ImplicitOps | Priv, em_clts, clts), N,
3226 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3227 N, D(ImplicitOps | ModRM), N, N,
3229 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3231 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3232 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3233 DIP(ModRM | SrcMem | Priv | Op3264, cr_write, check_cr_write),
3234 DIP(ModRM | SrcMem | Priv | Op3264, dr_write, check_dr_write),
3236 N, N, N, N, N, N, N, N,
3238 DI(ImplicitOps | Priv, wrmsr),
3239 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3240 DI(ImplicitOps | Priv, rdmsr),
3241 DIP(ImplicitOps | Priv, rdpmc, check_rdpmc),
3242 I(ImplicitOps | VendorSpecific, em_sysenter),
3243 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3245 N, N, N, N, N, N, N, N,
3247 X16(D(DstReg | SrcMem | ModRM | Mov)),
3249 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3254 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3259 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3263 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3265 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3266 DI(ImplicitOps, cpuid), D(DstMem | SrcReg | ModRM | BitOp),
3267 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3268 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
3270 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
3271 DI(ImplicitOps, rsm), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3272 D(DstMem | SrcReg | Src2ImmByte | ModRM),
3273 D(DstMem | SrcReg | Src2CL | ModRM),
3274 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
3276 D2bv(DstMem | SrcReg | ModRM | Lock),
3277 D(DstReg | SrcMemFAddr | ModRM), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3278 D(DstReg | SrcMemFAddr | ModRM), D(DstReg | SrcMemFAddr | ModRM),
3279 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3282 G(BitOp, group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
3283 D(DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
3284 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3286 D2bv(DstMem | SrcReg | ModRM | Lock),
3287 N, D(DstMem | SrcReg | ModRM | Mov),
3288 N, N, N, GD(0, &group9),
3289 N, N, N, N, N, N, N, N,
3291 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3293 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3295 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3311 static unsigned imm_size(struct decode_cache *c)
3315 size = (c->d & ByteOp) ? 1 : c->op_bytes;
3321 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3322 unsigned size, bool sign_extension)
3324 struct decode_cache *c = &ctxt->decode;
3325 int rc = X86EMUL_CONTINUE;
3329 op->addr.mem.ea = c->eip;
3330 /* NB. Immediates are sign-extended as necessary. */
3331 switch (op->bytes) {
3333 op->val = insn_fetch(s8, 1, c->eip);
3336 op->val = insn_fetch(s16, 2, c->eip);
3339 op->val = insn_fetch(s32, 4, c->eip);
3342 if (!sign_extension) {
3343 switch (op->bytes) {
3351 op->val &= 0xffffffff;
3359 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
3361 struct decode_cache *c = &ctxt->decode;
3362 int rc = X86EMUL_CONTINUE;
3363 int mode = ctxt->mode;
3364 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
3365 bool op_prefix = false;
3366 struct opcode opcode;
3367 struct operand memop = { .type = OP_NONE }, *memopp = NULL;
3370 c->fetch.start = c->eip;
3371 c->fetch.end = c->fetch.start + insn_len;
3373 memcpy(c->fetch.data, insn, insn_len);
3376 case X86EMUL_MODE_REAL:
3377 case X86EMUL_MODE_VM86:
3378 case X86EMUL_MODE_PROT16:
3379 def_op_bytes = def_ad_bytes = 2;
3381 case X86EMUL_MODE_PROT32:
3382 def_op_bytes = def_ad_bytes = 4;
3384 #ifdef CONFIG_X86_64
3385 case X86EMUL_MODE_PROT64:
3394 c->op_bytes = def_op_bytes;
3395 c->ad_bytes = def_ad_bytes;
3397 /* Legacy prefixes. */
3399 switch (c->b = insn_fetch(u8, 1, c->eip)) {
3400 case 0x66: /* operand-size override */
3402 /* switch between 2/4 bytes */
3403 c->op_bytes = def_op_bytes ^ 6;
3405 case 0x67: /* address-size override */
3406 if (mode == X86EMUL_MODE_PROT64)
3407 /* switch between 4/8 bytes */
3408 c->ad_bytes = def_ad_bytes ^ 12;
3410 /* switch between 2/4 bytes */
3411 c->ad_bytes = def_ad_bytes ^ 6;
3413 case 0x26: /* ES override */
3414 case 0x2e: /* CS override */
3415 case 0x36: /* SS override */
3416 case 0x3e: /* DS override */
3417 set_seg_override(c, (c->b >> 3) & 3);
3419 case 0x64: /* FS override */
3420 case 0x65: /* GS override */
3421 set_seg_override(c, c->b & 7);
3423 case 0x40 ... 0x4f: /* REX */
3424 if (mode != X86EMUL_MODE_PROT64)
3426 c->rex_prefix = c->b;
3428 case 0xf0: /* LOCK */
3431 case 0xf2: /* REPNE/REPNZ */
3432 case 0xf3: /* REP/REPE/REPZ */
3433 c->rep_prefix = c->b;
3439 /* Any legacy prefix after a REX prefix nullifies its effect. */
3447 if (c->rex_prefix & 8)
3448 c->op_bytes = 8; /* REX.W */
3450 /* Opcode byte(s). */
3451 opcode = opcode_table[c->b];
3452 /* Two-byte opcode? */
3455 c->b = insn_fetch(u8, 1, c->eip);
3456 opcode = twobyte_table[c->b];
3458 c->d = opcode.flags;
3460 while (c->d & GroupMask) {
3461 switch (c->d & GroupMask) {
3463 c->modrm = insn_fetch(u8, 1, c->eip);
3465 goffset = (c->modrm >> 3) & 7;
3466 opcode = opcode.u.group[goffset];
3469 c->modrm = insn_fetch(u8, 1, c->eip);
3471 goffset = (c->modrm >> 3) & 7;
3472 if ((c->modrm >> 6) == 3)
3473 opcode = opcode.u.gdual->mod3[goffset];
3475 opcode = opcode.u.gdual->mod012[goffset];
3478 goffset = c->modrm & 7;
3479 opcode = opcode.u.group[goffset];
3482 if (c->rep_prefix && op_prefix)
3483 return X86EMUL_UNHANDLEABLE;
3484 simd_prefix = op_prefix ? 0x66 : c->rep_prefix;
3485 switch (simd_prefix) {
3486 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
3487 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
3488 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
3489 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
3493 return X86EMUL_UNHANDLEABLE;
3497 c->d |= opcode.flags;
3500 c->execute = opcode.u.execute;
3501 c->check_perm = opcode.check_perm;
3502 c->intercept = opcode.intercept;
3505 if (c->d == 0 || (c->d & Undefined))
3508 if (!(c->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
3511 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
3514 if (c->d & Op3264) {
3515 if (mode == X86EMUL_MODE_PROT64)
3524 /* ModRM and SIB bytes. */
3526 rc = decode_modrm(ctxt, &memop);
3527 if (!c->has_seg_override)
3528 set_seg_override(c, c->modrm_seg);
3529 } else if (c->d & MemAbs)
3530 rc = decode_abs(ctxt, &memop);
3531 if (rc != X86EMUL_CONTINUE)
3534 if (!c->has_seg_override)
3535 set_seg_override(c, VCPU_SREG_DS);
3537 memop.addr.mem.seg = seg_override(ctxt, c);
3539 if (memop.type == OP_MEM && c->ad_bytes != 8)
3540 memop.addr.mem.ea = (u32)memop.addr.mem.ea;
3543 * Decode and fetch the source operand: register, memory
3546 switch (c->d & SrcMask) {
3550 decode_register_operand(ctxt, &c->src, c, 0);
3559 memop.bytes = (c->d & ByteOp) ? 1 :
3566 rc = decode_imm(ctxt, &c->src, 2, false);
3569 rc = decode_imm(ctxt, &c->src, imm_size(c), true);
3572 rc = decode_imm(ctxt, &c->src, imm_size(c), false);
3575 rc = decode_imm(ctxt, &c->src, 1, true);
3578 rc = decode_imm(ctxt, &c->src, 1, false);
3581 c->src.type = OP_REG;
3582 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3583 c->src.addr.reg = &c->regs[VCPU_REGS_RAX];
3584 fetch_register_operand(&c->src);
3591 c->src.type = OP_MEM;
3592 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3593 c->src.addr.mem.ea =
3594 register_address(c, c->regs[VCPU_REGS_RSI]);
3595 c->src.addr.mem.seg = seg_override(ctxt, c);
3599 c->src.type = OP_IMM;
3600 c->src.addr.mem.ea = c->eip;
3601 c->src.bytes = c->op_bytes + 2;
3602 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
3605 memop.bytes = c->op_bytes + 2;
3609 c->src.type = OP_REG;
3611 c->src.addr.reg = &c->regs[VCPU_REGS_RDX];
3612 fetch_register_operand(&c->src);
3616 if (rc != X86EMUL_CONTINUE)
3620 * Decode and fetch the second source operand: register, memory
3623 switch (c->d & Src2Mask) {
3628 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
3631 rc = decode_imm(ctxt, &c->src2, 1, true);
3638 rc = decode_imm(ctxt, &c->src2, imm_size(c), true);
3642 if (rc != X86EMUL_CONTINUE)
3645 /* Decode and fetch the destination operand: register or memory. */
3646 switch (c->d & DstMask) {
3648 decode_register_operand(ctxt, &c->dst, c,
3649 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
3652 c->dst.type = OP_IMM;
3653 c->dst.addr.mem.ea = c->eip;
3655 c->dst.val = insn_fetch(u8, 1, c->eip);
3661 if ((c->d & DstMask) == DstMem64)
3664 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3666 fetch_bit_operand(c);
3667 c->dst.orig_val = c->dst.val;
3670 c->dst.type = OP_REG;
3671 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3672 c->dst.addr.reg = &c->regs[VCPU_REGS_RAX];
3673 fetch_register_operand(&c->dst);
3674 c->dst.orig_val = c->dst.val;
3677 c->dst.type = OP_MEM;
3678 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
3679 c->dst.addr.mem.ea =
3680 register_address(c, c->regs[VCPU_REGS_RDI]);
3681 c->dst.addr.mem.seg = VCPU_SREG_ES;
3685 c->dst.type = OP_REG;
3687 c->dst.addr.reg = &c->regs[VCPU_REGS_RDX];
3688 fetch_register_operand(&c->dst);
3691 /* Special instructions do their own operand decoding. */
3693 c->dst.type = OP_NONE; /* Disable writeback. */
3698 if (memopp && memopp->type == OP_MEM && c->rip_relative)
3699 memopp->addr.mem.ea += c->eip;
3701 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3704 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
3706 struct decode_cache *c = &ctxt->decode;
3708 /* The second termination condition only applies for REPE
3709 * and REPNE. Test if the repeat string operation prefix is
3710 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
3711 * corresponding termination condition according to:
3712 * - if REPE/REPZ and ZF = 0 then done
3713 * - if REPNE/REPNZ and ZF = 1 then done
3715 if (((c->b == 0xa6) || (c->b == 0xa7) ||
3716 (c->b == 0xae) || (c->b == 0xaf))
3717 && (((c->rep_prefix == REPE_PREFIX) &&
3718 ((ctxt->eflags & EFLG_ZF) == 0))
3719 || ((c->rep_prefix == REPNE_PREFIX) &&
3720 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
3726 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
3728 struct x86_emulate_ops *ops = ctxt->ops;
3730 struct decode_cache *c = &ctxt->decode;
3731 int rc = X86EMUL_CONTINUE;
3732 int saved_dst_type = c->dst.type;
3733 int irq; /* Used for int 3, int, and into */
3735 c->mem_read.pos = 0;
3737 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
3738 rc = emulate_ud(ctxt);
3742 /* LOCK prefix is allowed only with some instructions */
3743 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
3744 rc = emulate_ud(ctxt);
3748 if ((c->d & SrcMask) == SrcMemFAddr && c->src.type != OP_MEM) {
3749 rc = emulate_ud(ctxt);
3754 && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)
3755 || !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
3756 rc = emulate_ud(ctxt);
3760 if ((c->d & Sse) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
3761 rc = emulate_nm(ctxt);
3765 if (unlikely(ctxt->guest_mode) && c->intercept) {
3766 rc = emulator_check_intercept(ctxt, c->intercept,
3767 X86_ICPT_PRE_EXCEPT);
3768 if (rc != X86EMUL_CONTINUE)
3772 /* Privileged instruction can be executed only in CPL=0 */
3773 if ((c->d & Priv) && ops->cpl(ctxt)) {
3774 rc = emulate_gp(ctxt, 0);
3778 /* Instruction can only be executed in protected mode */
3779 if ((c->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
3780 rc = emulate_ud(ctxt);
3784 /* Do instruction specific permission checks */
3785 if (c->check_perm) {
3786 rc = c->check_perm(ctxt);
3787 if (rc != X86EMUL_CONTINUE)
3791 if (unlikely(ctxt->guest_mode) && c->intercept) {
3792 rc = emulator_check_intercept(ctxt, c->intercept,
3793 X86_ICPT_POST_EXCEPT);
3794 if (rc != X86EMUL_CONTINUE)
3798 if (c->rep_prefix && (c->d & String)) {
3799 /* All REP prefixes have the same first termination condition */
3800 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
3806 if ((c->src.type == OP_MEM) && !(c->d & NoAccess)) {
3807 rc = segmented_read(ctxt, c->src.addr.mem,
3808 c->src.valptr, c->src.bytes);
3809 if (rc != X86EMUL_CONTINUE)
3811 c->src.orig_val64 = c->src.val64;
3814 if (c->src2.type == OP_MEM) {
3815 rc = segmented_read(ctxt, c->src2.addr.mem,
3816 &c->src2.val, c->src2.bytes);
3817 if (rc != X86EMUL_CONTINUE)
3821 if ((c->d & DstMask) == ImplicitOps)
3825 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
3826 /* optimisation - avoid slow emulated read if Mov */
3827 rc = segmented_read(ctxt, c->dst.addr.mem,
3828 &c->dst.val, c->dst.bytes);
3829 if (rc != X86EMUL_CONTINUE)
3832 c->dst.orig_val = c->dst.val;
3836 if (unlikely(ctxt->guest_mode) && c->intercept) {
3837 rc = emulator_check_intercept(ctxt, c->intercept,
3838 X86_ICPT_POST_MEMACCESS);
3839 if (rc != X86EMUL_CONTINUE)
3844 rc = c->execute(ctxt);
3845 if (rc != X86EMUL_CONTINUE)
3854 case 0x06: /* push es */
3855 rc = emulate_push_sreg(ctxt, VCPU_SREG_ES);
3857 case 0x07: /* pop es */
3858 rc = emulate_pop_sreg(ctxt, VCPU_SREG_ES);
3860 case 0x0e: /* push cs */
3861 rc = emulate_push_sreg(ctxt, VCPU_SREG_CS);
3863 case 0x16: /* push ss */
3864 rc = emulate_push_sreg(ctxt, VCPU_SREG_SS);
3866 case 0x17: /* pop ss */
3867 rc = emulate_pop_sreg(ctxt, VCPU_SREG_SS);
3869 case 0x1e: /* push ds */
3870 rc = emulate_push_sreg(ctxt, VCPU_SREG_DS);
3872 case 0x1f: /* pop ds */
3873 rc = emulate_pop_sreg(ctxt, VCPU_SREG_DS);
3875 case 0x40 ... 0x47: /* inc r16/r32 */
3876 emulate_1op("inc", c->dst, ctxt->eflags);
3878 case 0x48 ... 0x4f: /* dec r16/r32 */
3879 emulate_1op("dec", c->dst, ctxt->eflags);
3881 case 0x63: /* movsxd */
3882 if (ctxt->mode != X86EMUL_MODE_PROT64)
3883 goto cannot_emulate;
3884 c->dst.val = (s32) c->src.val;
3886 case 0x6c: /* insb */
3887 case 0x6d: /* insw/insd */
3888 c->src.val = c->regs[VCPU_REGS_RDX];
3890 case 0x6e: /* outsb */
3891 case 0x6f: /* outsw/outsd */
3892 c->dst.val = c->regs[VCPU_REGS_RDX];
3895 case 0x70 ... 0x7f: /* jcc (short) */
3896 if (test_cc(c->b, ctxt->eflags))
3897 jmp_rel(c, c->src.val);
3899 case 0x8c: /* mov r/m, sreg */
3900 if (c->modrm_reg > VCPU_SREG_GS) {
3901 rc = emulate_ud(ctxt);
3904 c->dst.val = get_segment_selector(ctxt, c->modrm_reg);
3906 case 0x8d: /* lea r16/r32, m */
3907 c->dst.val = c->src.addr.mem.ea;
3909 case 0x8e: { /* mov seg, r/m16 */
3914 if (c->modrm_reg == VCPU_SREG_CS ||
3915 c->modrm_reg > VCPU_SREG_GS) {
3916 rc = emulate_ud(ctxt);
3920 if (c->modrm_reg == VCPU_SREG_SS)
3921 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3923 rc = load_segment_descriptor(ctxt, sel, c->modrm_reg);
3925 c->dst.type = OP_NONE; /* Disable writeback. */
3928 case 0x8f: /* pop (sole member of Grp1a) */
3929 rc = em_grp1a(ctxt);
3931 case 0x90 ... 0x97: /* nop / xchg reg, rax */
3932 if (c->dst.addr.reg == &c->regs[VCPU_REGS_RAX])
3936 case 0x98: /* cbw/cwde/cdqe */
3937 switch (c->op_bytes) {
3938 case 2: c->dst.val = (s8)c->dst.val; break;
3939 case 4: c->dst.val = (s16)c->dst.val; break;
3940 case 8: c->dst.val = (s32)c->dst.val; break;
3946 case 0xc3: /* ret */
3947 c->dst.type = OP_REG;
3948 c->dst.addr.reg = &c->eip;
3949 c->dst.bytes = c->op_bytes;
3952 case 0xc4: /* les */
3953 rc = emulate_load_segment(ctxt, VCPU_SREG_ES);
3955 case 0xc5: /* lds */
3956 rc = emulate_load_segment(ctxt, VCPU_SREG_DS);
3958 case 0xcc: /* int3 */
3961 case 0xcd: /* int n */
3964 rc = emulate_int(ctxt, irq);
3966 case 0xce: /* into */
3967 if (ctxt->eflags & EFLG_OF) {
3972 case 0xd0 ... 0xd1: /* Grp2 */
3975 case 0xd2 ... 0xd3: /* Grp2 */
3976 c->src.val = c->regs[VCPU_REGS_RCX];
3979 case 0xe0 ... 0xe2: /* loop/loopz/loopnz */
3980 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3981 if (address_mask(c, c->regs[VCPU_REGS_RCX]) != 0 &&
3982 (c->b == 0xe2 || test_cc(c->b ^ 0x5, ctxt->eflags)))
3983 jmp_rel(c, c->src.val);
3985 case 0xe3: /* jcxz/jecxz/jrcxz */
3986 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0)
3987 jmp_rel(c, c->src.val);
3989 case 0xe4: /* inb */
3992 case 0xe6: /* outb */
3993 case 0xe7: /* out */
3995 case 0xe8: /* call (near) */ {
3996 long int rel = c->src.val;
3997 c->src.val = (unsigned long) c->eip;
4002 case 0xe9: /* jmp rel */
4003 case 0xeb: /* jmp rel short */
4004 jmp_rel(c, c->src.val);
4005 c->dst.type = OP_NONE; /* Disable writeback. */
4007 case 0xec: /* in al,dx */
4008 case 0xed: /* in (e/r)ax,dx */
4010 if (!pio_in_emulated(ctxt, c->dst.bytes, c->src.val,
4012 goto done; /* IO is needed */
4014 case 0xee: /* out dx,al */
4015 case 0xef: /* out dx,(e/r)ax */
4017 ops->pio_out_emulated(ctxt, c->src.bytes, c->dst.val,
4019 c->dst.type = OP_NONE; /* Disable writeback. */
4021 case 0xf4: /* hlt */
4022 ctxt->ops->halt(ctxt);
4024 case 0xf5: /* cmc */
4025 /* complement carry flag from eflags reg */
4026 ctxt->eflags ^= EFLG_CF;
4028 case 0xf6 ... 0xf7: /* Grp3 */
4031 case 0xf8: /* clc */
4032 ctxt->eflags &= ~EFLG_CF;
4034 case 0xf9: /* stc */
4035 ctxt->eflags |= EFLG_CF;
4037 case 0xfa: /* cli */
4038 if (emulator_bad_iopl(ctxt)) {
4039 rc = emulate_gp(ctxt, 0);
4042 ctxt->eflags &= ~X86_EFLAGS_IF;
4044 case 0xfb: /* sti */
4045 if (emulator_bad_iopl(ctxt)) {
4046 rc = emulate_gp(ctxt, 0);
4049 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
4050 ctxt->eflags |= X86_EFLAGS_IF;
4053 case 0xfc: /* cld */
4054 ctxt->eflags &= ~EFLG_DF;
4056 case 0xfd: /* std */
4057 ctxt->eflags |= EFLG_DF;
4059 case 0xfe: /* Grp4 */
4060 rc = em_grp45(ctxt);
4062 case 0xff: /* Grp5 */
4063 rc = em_grp45(ctxt);
4066 goto cannot_emulate;
4069 if (rc != X86EMUL_CONTINUE)
4073 rc = writeback(ctxt);
4074 if (rc != X86EMUL_CONTINUE)
4078 * restore dst type in case the decoding will be reused
4079 * (happens for string instruction )
4081 c->dst.type = saved_dst_type;
4083 if ((c->d & SrcMask) == SrcSI)
4084 string_addr_inc(ctxt, seg_override(ctxt, c),
4085 VCPU_REGS_RSI, &c->src);
4087 if ((c->d & DstMask) == DstDI)
4088 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
4091 if (c->rep_prefix && (c->d & String)) {
4092 struct read_cache *r = &c->io_read;
4093 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
4095 if (!string_insn_completed(ctxt)) {
4097 * Re-enter guest when pio read ahead buffer is empty
4098 * or, if it is not used, after each 1024 iteration.
4100 if ((r->end != 0 || c->regs[VCPU_REGS_RCX] & 0x3ff) &&
4101 (r->end == 0 || r->end != r->pos)) {
4103 * Reset read cache. Usually happens before
4104 * decode, but since instruction is restarted
4105 * we have to do it here.
4107 c->mem_read.end = 0;
4108 return EMULATION_RESTART;
4110 goto done; /* skip rip writeback */
4117 if (rc == X86EMUL_PROPAGATE_FAULT)
4118 ctxt->have_exception = true;
4119 if (rc == X86EMUL_INTERCEPTED)
4120 return EMULATION_INTERCEPTED;
4122 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4126 case 0x09: /* wbinvd */
4127 (ctxt->ops->wbinvd)(ctxt);
4129 case 0x08: /* invd */
4130 case 0x0d: /* GrpP (prefetch) */
4131 case 0x18: /* Grp16 (prefetch/nop) */
4133 case 0x20: /* mov cr, reg */
4134 c->dst.val = ops->get_cr(ctxt, c->modrm_reg);
4136 case 0x21: /* mov from dr to reg */
4137 ops->get_dr(ctxt, c->modrm_reg, &c->dst.val);
4139 case 0x22: /* mov reg, cr */
4140 if (ops->set_cr(ctxt, c->modrm_reg, c->src.val)) {
4141 emulate_gp(ctxt, 0);
4142 rc = X86EMUL_PROPAGATE_FAULT;
4145 c->dst.type = OP_NONE;
4147 case 0x23: /* mov from reg to dr */
4148 if (ops->set_dr(ctxt, c->modrm_reg, c->src.val &
4149 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
4150 ~0ULL : ~0U)) < 0) {
4151 /* #UD condition is already handled by the code above */
4152 emulate_gp(ctxt, 0);
4153 rc = X86EMUL_PROPAGATE_FAULT;
4157 c->dst.type = OP_NONE; /* no writeback */
4161 msr_data = (u32)c->regs[VCPU_REGS_RAX]
4162 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
4163 if (ops->set_msr(ctxt, c->regs[VCPU_REGS_RCX], msr_data)) {
4164 emulate_gp(ctxt, 0);
4165 rc = X86EMUL_PROPAGATE_FAULT;
4168 rc = X86EMUL_CONTINUE;
4172 if (ops->get_msr(ctxt, c->regs[VCPU_REGS_RCX], &msr_data)) {
4173 emulate_gp(ctxt, 0);
4174 rc = X86EMUL_PROPAGATE_FAULT;
4177 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
4178 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
4180 rc = X86EMUL_CONTINUE;
4182 case 0x40 ... 0x4f: /* cmov */
4183 c->dst.val = c->dst.orig_val = c->src.val;
4184 if (!test_cc(c->b, ctxt->eflags))
4185 c->dst.type = OP_NONE; /* no writeback */
4187 case 0x80 ... 0x8f: /* jnz rel, etc*/
4188 if (test_cc(c->b, ctxt->eflags))
4189 jmp_rel(c, c->src.val);
4191 case 0x90 ... 0x9f: /* setcc r/m8 */
4192 c->dst.val = test_cc(c->b, ctxt->eflags);
4194 case 0xa0: /* push fs */
4195 rc = emulate_push_sreg(ctxt, VCPU_SREG_FS);
4197 case 0xa1: /* pop fs */
4198 rc = emulate_pop_sreg(ctxt, VCPU_SREG_FS);
4202 c->dst.type = OP_NONE;
4203 /* only subword offset */
4204 c->src.val &= (c->dst.bytes << 3) - 1;
4205 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
4207 case 0xa4: /* shld imm8, r, r/m */
4208 case 0xa5: /* shld cl, r, r/m */
4209 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
4211 case 0xa8: /* push gs */
4212 rc = emulate_push_sreg(ctxt, VCPU_SREG_GS);
4214 case 0xa9: /* pop gs */
4215 rc = emulate_pop_sreg(ctxt, VCPU_SREG_GS);
4219 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
4221 case 0xac: /* shrd imm8, r, r/m */
4222 case 0xad: /* shrd cl, r, r/m */
4223 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
4225 case 0xae: /* clflush */
4227 case 0xb0 ... 0xb1: /* cmpxchg */
4229 * Save real source value, then compare EAX against
4232 c->src.orig_val = c->src.val;
4233 c->src.val = c->regs[VCPU_REGS_RAX];
4234 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
4235 if (ctxt->eflags & EFLG_ZF) {
4236 /* Success: write back to memory. */
4237 c->dst.val = c->src.orig_val;
4239 /* Failure: write the value we saw to EAX. */
4240 c->dst.type = OP_REG;
4241 c->dst.addr.reg = (unsigned long *)&c->regs[VCPU_REGS_RAX];
4244 case 0xb2: /* lss */
4245 rc = emulate_load_segment(ctxt, VCPU_SREG_SS);
4249 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
4251 case 0xb4: /* lfs */
4252 rc = emulate_load_segment(ctxt, VCPU_SREG_FS);
4254 case 0xb5: /* lgs */
4255 rc = emulate_load_segment(ctxt, VCPU_SREG_GS);
4257 case 0xb6 ... 0xb7: /* movzx */
4258 c->dst.bytes = c->op_bytes;
4259 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
4262 case 0xba: /* Grp8 */
4263 switch (c->modrm_reg & 3) {
4276 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
4278 case 0xbc: { /* bsf */
4280 __asm__ ("bsf %2, %0; setz %1"
4281 : "=r"(c->dst.val), "=q"(zf)
4283 ctxt->eflags &= ~X86_EFLAGS_ZF;
4285 ctxt->eflags |= X86_EFLAGS_ZF;
4286 c->dst.type = OP_NONE; /* Disable writeback. */
4290 case 0xbd: { /* bsr */
4292 __asm__ ("bsr %2, %0; setz %1"
4293 : "=r"(c->dst.val), "=q"(zf)
4295 ctxt->eflags &= ~X86_EFLAGS_ZF;
4297 ctxt->eflags |= X86_EFLAGS_ZF;
4298 c->dst.type = OP_NONE; /* Disable writeback. */
4302 case 0xbe ... 0xbf: /* movsx */
4303 c->dst.bytes = c->op_bytes;
4304 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
4307 case 0xc0 ... 0xc1: /* xadd */
4308 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
4309 /* Write back the register source. */
4310 c->src.val = c->dst.orig_val;
4311 write_register_operand(&c->src);
4313 case 0xc3: /* movnti */
4314 c->dst.bytes = c->op_bytes;
4315 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
4318 case 0xc7: /* Grp9 (cmpxchg8b) */
4322 goto cannot_emulate;
4325 if (rc != X86EMUL_CONTINUE)
4331 return EMULATION_FAILED;