1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpBits 5 /* Width of operand field */
66 #define OpMask ((1ull << OpBits) - 1)
69 * Opcode effective-address decode tables.
70 * Note that we only emulate instructions that have at least one memory
71 * operand (excluding implicit stack references). We assume that stack
72 * references and instruction fetches will never occur in special memory
73 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
77 /* Operand sizes: 8-bit operands or specified/overridden size. */
78 #define ByteOp (1<<0) /* 8-bit operands. */
79 /* Destination operand type. */
81 #define ImplicitOps (OpImplicit << DstShift)
82 #define DstReg (OpReg << DstShift)
83 #define DstMem (OpMem << DstShift)
84 #define DstAcc (OpAcc << DstShift)
85 #define DstDI (OpDI << DstShift)
86 #define DstMem64 (OpMem64 << DstShift)
87 #define DstImmUByte (OpImmUByte << DstShift)
88 #define DstDX (OpDX << DstShift)
89 #define DstMask (OpMask << DstShift)
90 /* Source operand type. */
92 #define SrcNone (OpNone << SrcShift)
93 #define SrcReg (OpReg << SrcShift)
94 #define SrcMem (OpMem << SrcShift)
95 #define SrcMem16 (OpMem16 << SrcShift)
96 #define SrcMem32 (OpMem32 << SrcShift)
97 #define SrcImm (OpImm << SrcShift)
98 #define SrcImmByte (OpImmByte << SrcShift)
99 #define SrcOne (OpOne << SrcShift)
100 #define SrcImmUByte (OpImmUByte << SrcShift)
101 #define SrcImmU (OpImmU << SrcShift)
102 #define SrcSI (OpSI << SrcShift)
103 #define SrcXLat (OpXLat << SrcShift)
104 #define SrcImmFAddr (OpImmFAddr << SrcShift)
105 #define SrcMemFAddr (OpMemFAddr << SrcShift)
106 #define SrcAcc (OpAcc << SrcShift)
107 #define SrcImmU16 (OpImmU16 << SrcShift)
108 #define SrcImm64 (OpImm64 << SrcShift)
109 #define SrcDX (OpDX << SrcShift)
110 #define SrcMem8 (OpMem8 << SrcShift)
111 #define SrcMask (OpMask << SrcShift)
112 #define BitOp (1<<11)
113 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
114 #define String (1<<13) /* String instruction (rep capable) */
115 #define Stack (1<<14) /* Stack instruction (push/pop) */
116 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
117 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
118 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
119 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
120 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
121 #define Escape (5<<15) /* Escape to coprocessor instruction */
122 #define Sse (1<<18) /* SSE Vector instruction */
123 /* Generic ModRM decode. */
124 #define ModRM (1<<19)
125 /* Destination is only written; never read. */
128 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
129 #define VendorSpecific (1<<22) /* Vendor specific instruction */
130 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132 #define Undefined (1<<25) /* No Such Instruction */
133 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
134 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
136 #define PageTable (1 << 29) /* instruction used to write page table */
137 #define NotImpl (1 << 30) /* instruction is not implemented */
138 /* Source 2 operand type */
139 #define Src2Shift (31)
140 #define Src2None (OpNone << Src2Shift)
141 #define Src2CL (OpCL << Src2Shift)
142 #define Src2ImmByte (OpImmByte << Src2Shift)
143 #define Src2One (OpOne << Src2Shift)
144 #define Src2Imm (OpImm << Src2Shift)
145 #define Src2ES (OpES << Src2Shift)
146 #define Src2CS (OpCS << Src2Shift)
147 #define Src2SS (OpSS << Src2Shift)
148 #define Src2DS (OpDS << Src2Shift)
149 #define Src2FS (OpFS << Src2Shift)
150 #define Src2GS (OpGS << Src2Shift)
151 #define Src2Mask (OpMask << Src2Shift)
152 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
153 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
154 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
155 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
156 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
157 #define NoWrite ((u64)1 << 45) /* No writeback */
159 #define X2(x...) x, x
160 #define X3(x...) X2(x), x
161 #define X4(x...) X2(x), X2(x)
162 #define X5(x...) X4(x), x
163 #define X6(x...) X4(x), X2(x)
164 #define X7(x...) X4(x), X3(x)
165 #define X8(x...) X4(x), X4(x)
166 #define X16(x...) X8(x), X8(x)
168 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
169 #define FASTOP_SIZE 8
172 * fastop functions have a special calling convention:
174 * dst: [rdx]:rax (in/out)
177 * flags: rflags (in/out)
179 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
180 * different operand sizes can be reached by calculation, rather than a jump
181 * table (which would be bigger than the code).
183 * fastop functions are declared as taking a never-defined fastop parameter,
184 * so they can't be called from C directly.
193 int (*execute)(struct x86_emulate_ctxt *ctxt);
194 const struct opcode *group;
195 const struct group_dual *gdual;
196 const struct gprefix *gprefix;
197 const struct escape *esc;
198 void (*fastop)(struct fastop *fake);
200 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
204 struct opcode mod012[8];
205 struct opcode mod3[8];
209 struct opcode pfx_no;
210 struct opcode pfx_66;
211 struct opcode pfx_f2;
212 struct opcode pfx_f3;
217 struct opcode high[64];
220 /* EFLAGS bit definitions. */
221 #define EFLG_ID (1<<21)
222 #define EFLG_VIP (1<<20)
223 #define EFLG_VIF (1<<19)
224 #define EFLG_AC (1<<18)
225 #define EFLG_VM (1<<17)
226 #define EFLG_RF (1<<16)
227 #define EFLG_IOPL (3<<12)
228 #define EFLG_NT (1<<14)
229 #define EFLG_OF (1<<11)
230 #define EFLG_DF (1<<10)
231 #define EFLG_IF (1<<9)
232 #define EFLG_TF (1<<8)
233 #define EFLG_SF (1<<7)
234 #define EFLG_ZF (1<<6)
235 #define EFLG_AF (1<<4)
236 #define EFLG_PF (1<<2)
237 #define EFLG_CF (1<<0)
239 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
240 #define EFLG_RESERVED_ONE_MASK 2
242 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
244 if (!(ctxt->regs_valid & (1 << nr))) {
245 ctxt->regs_valid |= 1 << nr;
246 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
248 return ctxt->_regs[nr];
251 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 ctxt->regs_valid |= 1 << nr;
254 ctxt->regs_dirty |= 1 << nr;
255 return &ctxt->_regs[nr];
258 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
261 return reg_write(ctxt, nr);
264 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
268 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
269 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
272 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
274 ctxt->regs_dirty = 0;
275 ctxt->regs_valid = 0;
279 * Instruction emulation:
280 * Most instructions are emulated directly via a fragment of inline assembly
281 * code. This allows us to save/restore EFLAGS and thus very easily pick up
282 * any modified flags.
285 #if defined(CONFIG_X86_64)
286 #define _LO32 "k" /* force 32-bit operand */
287 #define _STK "%%rsp" /* stack pointer */
288 #elif defined(__i386__)
289 #define _LO32 "" /* force 32-bit operand */
290 #define _STK "%%esp" /* stack pointer */
294 * These EFLAGS bits are restored from saved value during emulation, and
295 * any changes are written back to the saved value after emulation.
297 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 /* Before executing instruction: restore necessary bits in EFLAGS. */
300 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
301 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
302 "movl %"_sav",%"_LO32 _tmp"; " \
305 "movl %"_msk",%"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
308 "notl %"_LO32 _tmp"; " \
309 "andl %"_LO32 _tmp",("_STK"); " \
310 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
312 "orl %"_LO32 _tmp",("_STK"); " \
316 /* After executing instruction: write-back necessary bits in EFLAGS. */
317 #define _POST_EFLAGS(_sav, _msk, _tmp) \
318 /* _sav |= EFLAGS & _msk; */ \
321 "andl %"_msk",%"_LO32 _tmp"; " \
322 "orl %"_LO32 _tmp",%"_sav"; "
330 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "2") \
334 _op _suffix " %"_x"3,%1; " \
335 _POST_EFLAGS("0", "4", "2") \
336 : "=m" ((ctxt)->eflags), \
337 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
339 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
343 /* Raw emulation: instruction has two explicit operands. */
344 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
346 unsigned long _tmp; \
348 switch ((ctxt)->dst.bytes) { \
350 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
353 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
356 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
361 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
363 unsigned long _tmp; \
364 switch ((ctxt)->dst.bytes) { \
366 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
369 __emulate_2op_nobyte(ctxt, _op, \
370 _wx, _wy, _lx, _ly, _qx, _qy); \
375 /* Source operand is byte-sized and may be restricted to just %cl. */
376 #define emulate_2op_SrcB(ctxt, _op) \
377 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
379 /* Source operand is byte, word, long or quad sized. */
380 #define emulate_2op_SrcV(ctxt, _op) \
381 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
383 /* Source operand is word, long or quad sized. */
384 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
385 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
387 /* Instruction has three operands and one operand is stored in ECX register */
388 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
390 unsigned long _tmp; \
391 _type _clv = (ctxt)->src2.val; \
392 _type _srcv = (ctxt)->src.val; \
393 _type _dstv = (ctxt)->dst.val; \
395 __asm__ __volatile__ ( \
396 _PRE_EFLAGS("0", "5", "2") \
397 _op _suffix " %4,%1 \n" \
398 _POST_EFLAGS("0", "5", "2") \
399 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
400 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
403 (ctxt)->src2.val = (unsigned long) _clv; \
404 (ctxt)->src2.val = (unsigned long) _srcv; \
405 (ctxt)->dst.val = (unsigned long) _dstv; \
408 #define emulate_2op_cl(ctxt, _op) \
410 switch ((ctxt)->dst.bytes) { \
412 __emulate_2op_cl(ctxt, _op, "w", u16); \
415 __emulate_2op_cl(ctxt, _op, "l", u32); \
418 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
423 #define __emulate_1op(ctxt, _op, _suffix) \
425 unsigned long _tmp; \
427 __asm__ __volatile__ ( \
428 _PRE_EFLAGS("0", "3", "2") \
429 _op _suffix " %1; " \
430 _POST_EFLAGS("0", "3", "2") \
431 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
433 : "i" (EFLAGS_MASK)); \
436 /* Instruction has only one explicit operand (no source operand). */
437 #define emulate_1op(ctxt, _op) \
439 switch ((ctxt)->dst.bytes) { \
440 case 1: __emulate_1op(ctxt, _op, "b"); break; \
441 case 2: __emulate_1op(ctxt, _op, "w"); break; \
442 case 4: __emulate_1op(ctxt, _op, "l"); break; \
443 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
447 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
449 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
450 #define FOP_RET "ret \n\t"
452 #define FOP_START(op) \
453 extern void em_##op(struct fastop *fake); \
454 asm(".pushsection .text, \"ax\" \n\t" \
455 ".global em_" #op " \n\t" \
462 #define FOPNOP() FOP_ALIGN FOP_RET
464 #define FOP1E(op, dst) \
465 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
467 #define FASTOP1(op) \
472 ON64(FOP1E(op##q, rax)) \
475 #define FOP2E(op, dst, src) \
476 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
478 #define FASTOP2(op) \
480 FOP2E(op##b, al, bl) \
481 FOP2E(op##w, ax, bx) \
482 FOP2E(op##l, eax, ebx) \
483 ON64(FOP2E(op##q, rax, rbx)) \
486 /* 2 operand, word only */
487 #define FASTOP2W(op) \
490 FOP2E(op##w, ax, bx) \
491 FOP2E(op##l, eax, ebx) \
492 ON64(FOP2E(op##q, rax, rbx)) \
495 /* 2 operand, src is CL */
496 #define FASTOP2CL(op) \
498 FOP2E(op##b, al, cl) \
499 FOP2E(op##w, ax, cl) \
500 FOP2E(op##l, eax, cl) \
501 ON64(FOP2E(op##q, rax, cl)) \
504 #define FOP3E(op, dst, src, src2) \
505 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
507 /* 3-operand, word-only, src2=cl */
508 #define FASTOP3WCL(op) \
511 FOP3E(op##w, ax, bx, cl) \
512 FOP3E(op##l, eax, ebx, cl) \
513 ON64(FOP3E(op##q, rax, rbx, cl)) \
516 /* Special case for SETcc - 1 instruction per cc */
517 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
538 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
541 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
543 unsigned long _tmp; \
544 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
545 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
547 __asm__ __volatile__ ( \
548 _PRE_EFLAGS("0", "5", "1") \
550 _op _suffix " %6; " \
552 _POST_EFLAGS("0", "5", "1") \
553 ".pushsection .fixup,\"ax\" \n\t" \
554 "3: movb $1, %4 \n\t" \
557 _ASM_EXTABLE(1b, 3b) \
558 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
559 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
560 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
563 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
564 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
566 switch((ctxt)->src.bytes) { \
568 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
571 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
574 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
577 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
582 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
583 enum x86_intercept intercept,
584 enum x86_intercept_stage stage)
586 struct x86_instruction_info info = {
587 .intercept = intercept,
588 .rep_prefix = ctxt->rep_prefix,
589 .modrm_mod = ctxt->modrm_mod,
590 .modrm_reg = ctxt->modrm_reg,
591 .modrm_rm = ctxt->modrm_rm,
592 .src_val = ctxt->src.val64,
593 .src_bytes = ctxt->src.bytes,
594 .dst_bytes = ctxt->dst.bytes,
595 .ad_bytes = ctxt->ad_bytes,
596 .next_rip = ctxt->eip,
599 return ctxt->ops->intercept(ctxt, &info, stage);
602 static void assign_masked(ulong *dest, ulong src, ulong mask)
604 *dest = (*dest & ~mask) | (src & mask);
607 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
609 return (1UL << (ctxt->ad_bytes << 3)) - 1;
612 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
615 struct desc_struct ss;
617 if (ctxt->mode == X86EMUL_MODE_PROT64)
619 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
620 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
623 static int stack_size(struct x86_emulate_ctxt *ctxt)
625 return (__fls(stack_mask(ctxt)) + 1) >> 3;
628 /* Access/update address held in a register, based on addressing mode. */
629 static inline unsigned long
630 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
632 if (ctxt->ad_bytes == sizeof(unsigned long))
635 return reg & ad_mask(ctxt);
638 static inline unsigned long
639 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
641 return address_mask(ctxt, reg);
644 static void masked_increment(ulong *reg, ulong mask, int inc)
646 assign_masked(reg, *reg + inc, mask);
650 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
654 if (ctxt->ad_bytes == sizeof(unsigned long))
657 mask = ad_mask(ctxt);
658 masked_increment(reg, mask, inc);
661 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
663 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
666 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
668 register_address_increment(ctxt, &ctxt->_eip, rel);
671 static u32 desc_limit_scaled(struct desc_struct *desc)
673 u32 limit = get_desc_limit(desc);
675 return desc->g ? (limit << 12) | 0xfff : limit;
678 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
680 ctxt->has_seg_override = true;
681 ctxt->seg_override = seg;
684 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
686 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
689 return ctxt->ops->get_cached_segment_base(ctxt, seg);
692 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
694 if (!ctxt->has_seg_override)
697 return ctxt->seg_override;
700 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
701 u32 error, bool valid)
703 ctxt->exception.vector = vec;
704 ctxt->exception.error_code = error;
705 ctxt->exception.error_code_valid = valid;
706 return X86EMUL_PROPAGATE_FAULT;
709 static int emulate_db(struct x86_emulate_ctxt *ctxt)
711 return emulate_exception(ctxt, DB_VECTOR, 0, false);
714 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
716 return emulate_exception(ctxt, GP_VECTOR, err, true);
719 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
721 return emulate_exception(ctxt, SS_VECTOR, err, true);
724 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
726 return emulate_exception(ctxt, UD_VECTOR, 0, false);
729 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
731 return emulate_exception(ctxt, TS_VECTOR, err, true);
734 static int emulate_de(struct x86_emulate_ctxt *ctxt)
736 return emulate_exception(ctxt, DE_VECTOR, 0, false);
739 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
741 return emulate_exception(ctxt, NM_VECTOR, 0, false);
744 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
747 struct desc_struct desc;
749 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
753 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
758 struct desc_struct desc;
760 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
761 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
765 * x86 defines three classes of vector instructions: explicitly
766 * aligned, explicitly unaligned, and the rest, which change behaviour
767 * depending on whether they're AVX encoded or not.
769 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
770 * subject to the same check.
772 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
774 if (likely(size < 16))
777 if (ctxt->d & Aligned)
779 else if (ctxt->d & Unaligned)
781 else if (ctxt->d & Avx)
787 static int __linearize(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 unsigned size, bool write, bool fetch,
792 struct desc_struct desc;
799 la = seg_base(ctxt, addr.seg) + addr.ea;
800 switch (ctxt->mode) {
801 case X86EMUL_MODE_PROT64:
802 if (((signed long)la << 16) >> 16 != la)
803 return emulate_gp(ctxt, 0);
806 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
810 /* code segment in protected mode or read-only data segment */
811 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
812 || !(desc.type & 2)) && write)
814 /* unreadable code segment */
815 if (!fetch && (desc.type & 8) && !(desc.type & 2))
817 lim = desc_limit_scaled(&desc);
818 if ((desc.type & 8) || !(desc.type & 4)) {
819 /* expand-up segment */
820 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
823 /* expand-down segment */
824 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
826 lim = desc.d ? 0xffffffff : 0xffff;
827 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
830 cpl = ctxt->ops->cpl(ctxt);
831 if (!(desc.type & 8)) {
835 } else if ((desc.type & 8) && !(desc.type & 4)) {
836 /* nonconforming code segment */
839 } else if ((desc.type & 8) && (desc.type & 4)) {
840 /* conforming code segment */
846 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
848 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
849 return emulate_gp(ctxt, 0);
851 return X86EMUL_CONTINUE;
853 if (addr.seg == VCPU_SREG_SS)
854 return emulate_ss(ctxt, sel);
856 return emulate_gp(ctxt, sel);
859 static int linearize(struct x86_emulate_ctxt *ctxt,
860 struct segmented_address addr,
861 unsigned size, bool write,
864 return __linearize(ctxt, addr, size, write, false, linear);
868 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
869 struct segmented_address addr,
876 rc = linearize(ctxt, addr, size, false, &linear);
877 if (rc != X86EMUL_CONTINUE)
879 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
883 * Fetch the next byte of the instruction being emulated which is pointed to
884 * by ctxt->_eip, then increment ctxt->_eip.
886 * Also prefetch the remaining bytes of the instruction without crossing page
887 * boundary if they are not in fetch_cache yet.
889 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
891 struct fetch_cache *fc = &ctxt->fetch;
895 if (ctxt->_eip == fc->end) {
896 unsigned long linear;
897 struct segmented_address addr = { .seg = VCPU_SREG_CS,
899 cur_size = fc->end - fc->start;
900 size = min(15UL - cur_size,
901 PAGE_SIZE - offset_in_page(ctxt->_eip));
902 rc = __linearize(ctxt, addr, size, false, true, &linear);
903 if (unlikely(rc != X86EMUL_CONTINUE))
905 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
906 size, &ctxt->exception);
907 if (unlikely(rc != X86EMUL_CONTINUE))
911 *dest = fc->data[ctxt->_eip - fc->start];
913 return X86EMUL_CONTINUE;
916 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
917 void *dest, unsigned size)
921 /* x86 instructions are limited to 15 bytes. */
922 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
923 return X86EMUL_UNHANDLEABLE;
925 rc = do_insn_fetch_byte(ctxt, dest++);
926 if (rc != X86EMUL_CONTINUE)
929 return X86EMUL_CONTINUE;
932 /* Fetch next part of the instruction being emulated. */
933 #define insn_fetch(_type, _ctxt) \
934 ({ unsigned long _x; \
935 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
936 if (rc != X86EMUL_CONTINUE) \
941 #define insn_fetch_arr(_arr, _size, _ctxt) \
942 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
943 if (rc != X86EMUL_CONTINUE) \
948 * Given the 'reg' portion of a ModRM byte, and a register block, return a
949 * pointer into the block that addresses the relevant register.
950 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
952 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
957 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
958 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
960 p = reg_rmw(ctxt, modrm_reg);
964 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
965 struct segmented_address addr,
966 u16 *size, unsigned long *address, int op_bytes)
973 rc = segmented_read_std(ctxt, addr, size, 2);
974 if (rc != X86EMUL_CONTINUE)
977 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1016 static u8 test_cc(unsigned int condition, unsigned long flags)
1019 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1021 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1022 asm("push %[flags]; popf; call *%[fastop]"
1023 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1027 static void fetch_register_operand(struct operand *op)
1029 switch (op->bytes) {
1031 op->val = *(u8 *)op->addr.reg;
1034 op->val = *(u16 *)op->addr.reg;
1037 op->val = *(u32 *)op->addr.reg;
1040 op->val = *(u64 *)op->addr.reg;
1045 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1047 ctxt->ops->get_fpu(ctxt);
1049 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1050 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1051 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1052 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1053 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1054 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1055 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1056 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1057 #ifdef CONFIG_X86_64
1058 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1059 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1060 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1061 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1062 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1063 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1064 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1065 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1069 ctxt->ops->put_fpu(ctxt);
1072 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1075 ctxt->ops->get_fpu(ctxt);
1077 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1078 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1079 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1080 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1081 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1082 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1083 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1084 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1085 #ifdef CONFIG_X86_64
1086 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1087 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1088 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1089 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1090 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1091 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1092 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1093 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1097 ctxt->ops->put_fpu(ctxt);
1100 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1102 ctxt->ops->get_fpu(ctxt);
1104 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1105 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1106 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1107 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1108 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1109 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1110 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1111 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1114 ctxt->ops->put_fpu(ctxt);
1117 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1119 ctxt->ops->get_fpu(ctxt);
1121 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1122 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1123 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1124 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1125 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1126 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1127 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1128 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1131 ctxt->ops->put_fpu(ctxt);
1134 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1136 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1137 return emulate_nm(ctxt);
1139 ctxt->ops->get_fpu(ctxt);
1140 asm volatile("fninit");
1141 ctxt->ops->put_fpu(ctxt);
1142 return X86EMUL_CONTINUE;
1145 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1149 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1150 return emulate_nm(ctxt);
1152 ctxt->ops->get_fpu(ctxt);
1153 asm volatile("fnstcw %0": "+m"(fcw));
1154 ctxt->ops->put_fpu(ctxt);
1156 /* force 2 byte destination */
1157 ctxt->dst.bytes = 2;
1158 ctxt->dst.val = fcw;
1160 return X86EMUL_CONTINUE;
1163 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1167 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1168 return emulate_nm(ctxt);
1170 ctxt->ops->get_fpu(ctxt);
1171 asm volatile("fnstsw %0": "+m"(fsw));
1172 ctxt->ops->put_fpu(ctxt);
1174 /* force 2 byte destination */
1175 ctxt->dst.bytes = 2;
1176 ctxt->dst.val = fsw;
1178 return X86EMUL_CONTINUE;
1181 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1184 unsigned reg = ctxt->modrm_reg;
1185 int highbyte_regs = ctxt->rex_prefix == 0;
1187 if (!(ctxt->d & ModRM))
1188 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1190 if (ctxt->d & Sse) {
1194 read_sse_reg(ctxt, &op->vec_val, reg);
1197 if (ctxt->d & Mmx) {
1206 if (ctxt->d & ByteOp) {
1207 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1210 op->addr.reg = decode_register(ctxt, reg, 0);
1211 op->bytes = ctxt->op_bytes;
1213 fetch_register_operand(op);
1214 op->orig_val = op->val;
1217 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1219 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1220 ctxt->modrm_seg = VCPU_SREG_SS;
1223 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1227 int index_reg = 0, base_reg = 0, scale;
1228 int rc = X86EMUL_CONTINUE;
1231 if (ctxt->rex_prefix) {
1232 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1233 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1234 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1237 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1238 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1239 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1240 ctxt->modrm_seg = VCPU_SREG_DS;
1242 if (ctxt->modrm_mod == 3) {
1244 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1245 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1246 if (ctxt->d & Sse) {
1249 op->addr.xmm = ctxt->modrm_rm;
1250 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1253 if (ctxt->d & Mmx) {
1256 op->addr.xmm = ctxt->modrm_rm & 7;
1259 fetch_register_operand(op);
1265 if (ctxt->ad_bytes == 2) {
1266 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1267 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1268 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1269 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1271 /* 16-bit ModR/M decode. */
1272 switch (ctxt->modrm_mod) {
1274 if (ctxt->modrm_rm == 6)
1275 modrm_ea += insn_fetch(u16, ctxt);
1278 modrm_ea += insn_fetch(s8, ctxt);
1281 modrm_ea += insn_fetch(u16, ctxt);
1284 switch (ctxt->modrm_rm) {
1286 modrm_ea += bx + si;
1289 modrm_ea += bx + di;
1292 modrm_ea += bp + si;
1295 modrm_ea += bp + di;
1304 if (ctxt->modrm_mod != 0)
1311 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1312 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1313 ctxt->modrm_seg = VCPU_SREG_SS;
1314 modrm_ea = (u16)modrm_ea;
1316 /* 32/64-bit ModR/M decode. */
1317 if ((ctxt->modrm_rm & 7) == 4) {
1318 sib = insn_fetch(u8, ctxt);
1319 index_reg |= (sib >> 3) & 7;
1320 base_reg |= sib & 7;
1323 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1324 modrm_ea += insn_fetch(s32, ctxt);
1326 modrm_ea += reg_read(ctxt, base_reg);
1327 adjust_modrm_seg(ctxt, base_reg);
1330 modrm_ea += reg_read(ctxt, index_reg) << scale;
1331 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1332 if (ctxt->mode == X86EMUL_MODE_PROT64)
1333 ctxt->rip_relative = 1;
1335 base_reg = ctxt->modrm_rm;
1336 modrm_ea += reg_read(ctxt, base_reg);
1337 adjust_modrm_seg(ctxt, base_reg);
1339 switch (ctxt->modrm_mod) {
1341 if (ctxt->modrm_rm == 5)
1342 modrm_ea += insn_fetch(s32, ctxt);
1345 modrm_ea += insn_fetch(s8, ctxt);
1348 modrm_ea += insn_fetch(s32, ctxt);
1352 op->addr.mem.ea = modrm_ea;
1357 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1360 int rc = X86EMUL_CONTINUE;
1363 switch (ctxt->ad_bytes) {
1365 op->addr.mem.ea = insn_fetch(u16, ctxt);
1368 op->addr.mem.ea = insn_fetch(u32, ctxt);
1371 op->addr.mem.ea = insn_fetch(u64, ctxt);
1378 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1382 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1383 mask = ~(ctxt->dst.bytes * 8 - 1);
1385 if (ctxt->src.bytes == 2)
1386 sv = (s16)ctxt->src.val & (s16)mask;
1387 else if (ctxt->src.bytes == 4)
1388 sv = (s32)ctxt->src.val & (s32)mask;
1390 ctxt->dst.addr.mem.ea += (sv >> 3);
1393 /* only subword offset */
1394 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1397 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1398 unsigned long addr, void *dest, unsigned size)
1401 struct read_cache *mc = &ctxt->mem_read;
1403 if (mc->pos < mc->end)
1406 WARN_ON((mc->end + size) >= sizeof(mc->data));
1408 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1410 if (rc != X86EMUL_CONTINUE)
1416 memcpy(dest, mc->data + mc->pos, size);
1418 return X86EMUL_CONTINUE;
1421 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1422 struct segmented_address addr,
1429 rc = linearize(ctxt, addr, size, false, &linear);
1430 if (rc != X86EMUL_CONTINUE)
1432 return read_emulated(ctxt, linear, data, size);
1435 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1436 struct segmented_address addr,
1443 rc = linearize(ctxt, addr, size, true, &linear);
1444 if (rc != X86EMUL_CONTINUE)
1446 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1450 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1451 struct segmented_address addr,
1452 const void *orig_data, const void *data,
1458 rc = linearize(ctxt, addr, size, true, &linear);
1459 if (rc != X86EMUL_CONTINUE)
1461 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1462 size, &ctxt->exception);
1465 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1466 unsigned int size, unsigned short port,
1469 struct read_cache *rc = &ctxt->io_read;
1471 if (rc->pos == rc->end) { /* refill pio read ahead */
1472 unsigned int in_page, n;
1473 unsigned int count = ctxt->rep_prefix ?
1474 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1475 in_page = (ctxt->eflags & EFLG_DF) ?
1476 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1477 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1478 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1482 rc->pos = rc->end = 0;
1483 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1488 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1489 ctxt->dst.data = rc->data + rc->pos;
1490 ctxt->dst.type = OP_MEM_STR;
1491 ctxt->dst.count = (rc->end - rc->pos) / size;
1494 memcpy(dest, rc->data + rc->pos, size);
1500 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1501 u16 index, struct desc_struct *desc)
1506 ctxt->ops->get_idt(ctxt, &dt);
1508 if (dt.size < index * 8 + 7)
1509 return emulate_gp(ctxt, index << 3 | 0x2);
1511 addr = dt.address + index * 8;
1512 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1516 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1517 u16 selector, struct desc_ptr *dt)
1519 const struct x86_emulate_ops *ops = ctxt->ops;
1521 if (selector & 1 << 2) {
1522 struct desc_struct desc;
1525 memset (dt, 0, sizeof *dt);
1526 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1529 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1530 dt->address = get_desc_base(&desc);
1532 ops->get_gdt(ctxt, dt);
1535 /* allowed just for 8 bytes segments */
1536 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1537 u16 selector, struct desc_struct *desc,
1541 u16 index = selector >> 3;
1544 get_descriptor_table_ptr(ctxt, selector, &dt);
1546 if (dt.size < index * 8 + 7)
1547 return emulate_gp(ctxt, selector & 0xfffc);
1549 *desc_addr_p = addr = dt.address + index * 8;
1550 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1554 /* allowed just for 8 bytes segments */
1555 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1556 u16 selector, struct desc_struct *desc)
1559 u16 index = selector >> 3;
1562 get_descriptor_table_ptr(ctxt, selector, &dt);
1564 if (dt.size < index * 8 + 7)
1565 return emulate_gp(ctxt, selector & 0xfffc);
1567 addr = dt.address + index * 8;
1568 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1572 /* Does not support long mode */
1573 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1574 u16 selector, int seg)
1576 struct desc_struct seg_desc, old_desc;
1578 unsigned err_vec = GP_VECTOR;
1580 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1585 memset(&seg_desc, 0, sizeof seg_desc);
1587 if (ctxt->mode == X86EMUL_MODE_REAL) {
1588 /* set real mode segment descriptor (keep limit etc. for
1590 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1591 set_desc_base(&seg_desc, selector << 4);
1593 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1594 /* VM86 needs a clean new segment descriptor */
1595 set_desc_base(&seg_desc, selector << 4);
1596 set_desc_limit(&seg_desc, 0xffff);
1605 cpl = ctxt->ops->cpl(ctxt);
1607 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1608 if ((seg == VCPU_SREG_CS
1609 || (seg == VCPU_SREG_SS
1610 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1611 || seg == VCPU_SREG_TR)
1615 /* TR should be in GDT only */
1616 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1619 if (null_selector) /* for NULL selector skip all following checks */
1622 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1623 if (ret != X86EMUL_CONTINUE)
1626 err_code = selector & 0xfffc;
1627 err_vec = GP_VECTOR;
1629 /* can't load system descriptor into segment selector */
1630 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1634 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1643 * segment is not a writable data segment or segment
1644 * selector's RPL != CPL or segment selector's RPL != CPL
1646 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1650 if (!(seg_desc.type & 8))
1653 if (seg_desc.type & 4) {
1659 if (rpl > cpl || dpl != cpl)
1662 /* CS(RPL) <- CPL */
1663 selector = (selector & 0xfffc) | cpl;
1666 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1668 old_desc = seg_desc;
1669 seg_desc.type |= 2; /* busy */
1670 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1671 sizeof(seg_desc), &ctxt->exception);
1672 if (ret != X86EMUL_CONTINUE)
1675 case VCPU_SREG_LDTR:
1676 if (seg_desc.s || seg_desc.type != 2)
1679 default: /* DS, ES, FS, or GS */
1681 * segment is not a data or readable code segment or
1682 * ((segment is a data or nonconforming code segment)
1683 * and (both RPL and CPL > DPL))
1685 if ((seg_desc.type & 0xa) == 0x8 ||
1686 (((seg_desc.type & 0xc) != 0xc) &&
1687 (rpl > dpl && cpl > dpl)))
1693 /* mark segment as accessed */
1695 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1696 if (ret != X86EMUL_CONTINUE)
1700 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1701 return X86EMUL_CONTINUE;
1703 emulate_exception(ctxt, err_vec, err_code, true);
1704 return X86EMUL_PROPAGATE_FAULT;
1707 static void write_register_operand(struct operand *op)
1709 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1710 switch (op->bytes) {
1712 *(u8 *)op->addr.reg = (u8)op->val;
1715 *(u16 *)op->addr.reg = (u16)op->val;
1718 *op->addr.reg = (u32)op->val;
1719 break; /* 64b: zero-extend */
1721 *op->addr.reg = op->val;
1726 static int writeback(struct x86_emulate_ctxt *ctxt)
1730 if (ctxt->d & NoWrite)
1731 return X86EMUL_CONTINUE;
1733 switch (ctxt->dst.type) {
1735 write_register_operand(&ctxt->dst);
1738 if (ctxt->lock_prefix)
1739 rc = segmented_cmpxchg(ctxt,
1741 &ctxt->dst.orig_val,
1745 rc = segmented_write(ctxt,
1749 if (rc != X86EMUL_CONTINUE)
1753 rc = segmented_write(ctxt,
1756 ctxt->dst.bytes * ctxt->dst.count);
1757 if (rc != X86EMUL_CONTINUE)
1761 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1764 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1772 return X86EMUL_CONTINUE;
1775 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1777 struct segmented_address addr;
1779 rsp_increment(ctxt, -bytes);
1780 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1781 addr.seg = VCPU_SREG_SS;
1783 return segmented_write(ctxt, addr, data, bytes);
1786 static int em_push(struct x86_emulate_ctxt *ctxt)
1788 /* Disable writeback. */
1789 ctxt->dst.type = OP_NONE;
1790 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1793 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1794 void *dest, int len)
1797 struct segmented_address addr;
1799 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1800 addr.seg = VCPU_SREG_SS;
1801 rc = segmented_read(ctxt, addr, dest, len);
1802 if (rc != X86EMUL_CONTINUE)
1805 rsp_increment(ctxt, len);
1809 static int em_pop(struct x86_emulate_ctxt *ctxt)
1811 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1814 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1815 void *dest, int len)
1818 unsigned long val, change_mask;
1819 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1820 int cpl = ctxt->ops->cpl(ctxt);
1822 rc = emulate_pop(ctxt, &val, len);
1823 if (rc != X86EMUL_CONTINUE)
1826 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1827 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1829 switch(ctxt->mode) {
1830 case X86EMUL_MODE_PROT64:
1831 case X86EMUL_MODE_PROT32:
1832 case X86EMUL_MODE_PROT16:
1834 change_mask |= EFLG_IOPL;
1836 change_mask |= EFLG_IF;
1838 case X86EMUL_MODE_VM86:
1840 return emulate_gp(ctxt, 0);
1841 change_mask |= EFLG_IF;
1843 default: /* real mode */
1844 change_mask |= (EFLG_IOPL | EFLG_IF);
1848 *(unsigned long *)dest =
1849 (ctxt->eflags & ~change_mask) | (val & change_mask);
1854 static int em_popf(struct x86_emulate_ctxt *ctxt)
1856 ctxt->dst.type = OP_REG;
1857 ctxt->dst.addr.reg = &ctxt->eflags;
1858 ctxt->dst.bytes = ctxt->op_bytes;
1859 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1862 static int em_enter(struct x86_emulate_ctxt *ctxt)
1865 unsigned frame_size = ctxt->src.val;
1866 unsigned nesting_level = ctxt->src2.val & 31;
1870 return X86EMUL_UNHANDLEABLE;
1872 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1873 rc = push(ctxt, &rbp, stack_size(ctxt));
1874 if (rc != X86EMUL_CONTINUE)
1876 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1878 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1879 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1881 return X86EMUL_CONTINUE;
1884 static int em_leave(struct x86_emulate_ctxt *ctxt)
1886 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1888 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1891 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1893 int seg = ctxt->src2.val;
1895 ctxt->src.val = get_segment_selector(ctxt, seg);
1897 return em_push(ctxt);
1900 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1902 int seg = ctxt->src2.val;
1903 unsigned long selector;
1906 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1907 if (rc != X86EMUL_CONTINUE)
1910 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1914 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1916 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1917 int rc = X86EMUL_CONTINUE;
1918 int reg = VCPU_REGS_RAX;
1920 while (reg <= VCPU_REGS_RDI) {
1921 (reg == VCPU_REGS_RSP) ?
1922 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1925 if (rc != X86EMUL_CONTINUE)
1934 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1936 ctxt->src.val = (unsigned long)ctxt->eflags;
1937 return em_push(ctxt);
1940 static int em_popa(struct x86_emulate_ctxt *ctxt)
1942 int rc = X86EMUL_CONTINUE;
1943 int reg = VCPU_REGS_RDI;
1945 while (reg >= VCPU_REGS_RAX) {
1946 if (reg == VCPU_REGS_RSP) {
1947 rsp_increment(ctxt, ctxt->op_bytes);
1951 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1952 if (rc != X86EMUL_CONTINUE)
1959 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1961 const struct x86_emulate_ops *ops = ctxt->ops;
1968 /* TODO: Add limit checks */
1969 ctxt->src.val = ctxt->eflags;
1971 if (rc != X86EMUL_CONTINUE)
1974 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1976 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1978 if (rc != X86EMUL_CONTINUE)
1981 ctxt->src.val = ctxt->_eip;
1983 if (rc != X86EMUL_CONTINUE)
1986 ops->get_idt(ctxt, &dt);
1988 eip_addr = dt.address + (irq << 2);
1989 cs_addr = dt.address + (irq << 2) + 2;
1991 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1992 if (rc != X86EMUL_CONTINUE)
1995 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1996 if (rc != X86EMUL_CONTINUE)
1999 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2000 if (rc != X86EMUL_CONTINUE)
2008 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2012 invalidate_registers(ctxt);
2013 rc = __emulate_int_real(ctxt, irq);
2014 if (rc == X86EMUL_CONTINUE)
2015 writeback_registers(ctxt);
2019 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2021 switch(ctxt->mode) {
2022 case X86EMUL_MODE_REAL:
2023 return __emulate_int_real(ctxt, irq);
2024 case X86EMUL_MODE_VM86:
2025 case X86EMUL_MODE_PROT16:
2026 case X86EMUL_MODE_PROT32:
2027 case X86EMUL_MODE_PROT64:
2029 /* Protected mode interrupts unimplemented yet */
2030 return X86EMUL_UNHANDLEABLE;
2034 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2036 int rc = X86EMUL_CONTINUE;
2037 unsigned long temp_eip = 0;
2038 unsigned long temp_eflags = 0;
2039 unsigned long cs = 0;
2040 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2041 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2042 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2043 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2045 /* TODO: Add stack limit check */
2047 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2049 if (rc != X86EMUL_CONTINUE)
2052 if (temp_eip & ~0xffff)
2053 return emulate_gp(ctxt, 0);
2055 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2057 if (rc != X86EMUL_CONTINUE)
2060 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2062 if (rc != X86EMUL_CONTINUE)
2065 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2067 if (rc != X86EMUL_CONTINUE)
2070 ctxt->_eip = temp_eip;
2073 if (ctxt->op_bytes == 4)
2074 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2075 else if (ctxt->op_bytes == 2) {
2076 ctxt->eflags &= ~0xffff;
2077 ctxt->eflags |= temp_eflags;
2080 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2081 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2086 static int em_iret(struct x86_emulate_ctxt *ctxt)
2088 switch(ctxt->mode) {
2089 case X86EMUL_MODE_REAL:
2090 return emulate_iret_real(ctxt);
2091 case X86EMUL_MODE_VM86:
2092 case X86EMUL_MODE_PROT16:
2093 case X86EMUL_MODE_PROT32:
2094 case X86EMUL_MODE_PROT64:
2096 /* iret from protected mode unimplemented yet */
2097 return X86EMUL_UNHANDLEABLE;
2101 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2106 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2108 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2109 if (rc != X86EMUL_CONTINUE)
2113 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2114 return X86EMUL_CONTINUE;
2117 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2121 emulate_1op_rax_rdx(ctxt, "mul", ex);
2122 return X86EMUL_CONTINUE;
2125 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2129 emulate_1op_rax_rdx(ctxt, "imul", ex);
2130 return X86EMUL_CONTINUE;
2133 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2137 emulate_1op_rax_rdx(ctxt, "div", de);
2139 return emulate_de(ctxt);
2140 return X86EMUL_CONTINUE;
2143 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2147 emulate_1op_rax_rdx(ctxt, "idiv", de);
2149 return emulate_de(ctxt);
2150 return X86EMUL_CONTINUE;
2153 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2155 int rc = X86EMUL_CONTINUE;
2157 switch (ctxt->modrm_reg) {
2158 case 2: /* call near abs */ {
2160 old_eip = ctxt->_eip;
2161 ctxt->_eip = ctxt->src.val;
2162 ctxt->src.val = old_eip;
2166 case 4: /* jmp abs */
2167 ctxt->_eip = ctxt->src.val;
2169 case 5: /* jmp far */
2170 rc = em_jmp_far(ctxt);
2179 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2181 u64 old = ctxt->dst.orig_val64;
2183 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2184 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2185 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2186 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2187 ctxt->eflags &= ~EFLG_ZF;
2189 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2190 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2192 ctxt->eflags |= EFLG_ZF;
2194 return X86EMUL_CONTINUE;
2197 static int em_ret(struct x86_emulate_ctxt *ctxt)
2199 ctxt->dst.type = OP_REG;
2200 ctxt->dst.addr.reg = &ctxt->_eip;
2201 ctxt->dst.bytes = ctxt->op_bytes;
2202 return em_pop(ctxt);
2205 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2210 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2211 if (rc != X86EMUL_CONTINUE)
2213 if (ctxt->op_bytes == 4)
2214 ctxt->_eip = (u32)ctxt->_eip;
2215 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2216 if (rc != X86EMUL_CONTINUE)
2218 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2222 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2224 /* Save real source value, then compare EAX against destination. */
2225 ctxt->src.orig_val = ctxt->src.val;
2226 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2227 fastop(ctxt, em_cmp);
2229 if (ctxt->eflags & EFLG_ZF) {
2230 /* Success: write back to memory. */
2231 ctxt->dst.val = ctxt->src.orig_val;
2233 /* Failure: write the value we saw to EAX. */
2234 ctxt->dst.type = OP_REG;
2235 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2237 return X86EMUL_CONTINUE;
2240 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2242 int seg = ctxt->src2.val;
2246 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2248 rc = load_segment_descriptor(ctxt, sel, seg);
2249 if (rc != X86EMUL_CONTINUE)
2252 ctxt->dst.val = ctxt->src.val;
2257 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2258 struct desc_struct *cs, struct desc_struct *ss)
2260 cs->l = 0; /* will be adjusted later */
2261 set_desc_base(cs, 0); /* flat segment */
2262 cs->g = 1; /* 4kb granularity */
2263 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2264 cs->type = 0x0b; /* Read, Execute, Accessed */
2266 cs->dpl = 0; /* will be adjusted later */
2271 set_desc_base(ss, 0); /* flat segment */
2272 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2273 ss->g = 1; /* 4kb granularity */
2275 ss->type = 0x03; /* Read/Write, Accessed */
2276 ss->d = 1; /* 32bit stack segment */
2283 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2285 u32 eax, ebx, ecx, edx;
2288 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2289 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2290 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2291 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2294 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2296 const struct x86_emulate_ops *ops = ctxt->ops;
2297 u32 eax, ebx, ecx, edx;
2300 * syscall should always be enabled in longmode - so only become
2301 * vendor specific (cpuid) if other modes are active...
2303 if (ctxt->mode == X86EMUL_MODE_PROT64)
2308 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2310 * Intel ("GenuineIntel")
2311 * remark: Intel CPUs only support "syscall" in 64bit
2312 * longmode. Also an 64bit guest with a
2313 * 32bit compat-app running will #UD !! While this
2314 * behaviour can be fixed (by emulating) into AMD
2315 * response - CPUs of AMD can't behave like Intel.
2317 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2318 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2319 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2322 /* AMD ("AuthenticAMD") */
2323 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2324 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2325 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2328 /* AMD ("AMDisbetter!") */
2329 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2330 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2331 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2334 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2338 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2340 const struct x86_emulate_ops *ops = ctxt->ops;
2341 struct desc_struct cs, ss;
2346 /* syscall is not available in real mode */
2347 if (ctxt->mode == X86EMUL_MODE_REAL ||
2348 ctxt->mode == X86EMUL_MODE_VM86)
2349 return emulate_ud(ctxt);
2351 if (!(em_syscall_is_enabled(ctxt)))
2352 return emulate_ud(ctxt);
2354 ops->get_msr(ctxt, MSR_EFER, &efer);
2355 setup_syscalls_segments(ctxt, &cs, &ss);
2357 if (!(efer & EFER_SCE))
2358 return emulate_ud(ctxt);
2360 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2362 cs_sel = (u16)(msr_data & 0xfffc);
2363 ss_sel = (u16)(msr_data + 8);
2365 if (efer & EFER_LMA) {
2369 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2370 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2372 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2373 if (efer & EFER_LMA) {
2374 #ifdef CONFIG_X86_64
2375 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2378 ctxt->mode == X86EMUL_MODE_PROT64 ?
2379 MSR_LSTAR : MSR_CSTAR, &msr_data);
2380 ctxt->_eip = msr_data;
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~(msr_data | EFLG_RF);
2387 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2388 ctxt->_eip = (u32)msr_data;
2390 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2393 return X86EMUL_CONTINUE;
2396 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2398 const struct x86_emulate_ops *ops = ctxt->ops;
2399 struct desc_struct cs, ss;
2404 ops->get_msr(ctxt, MSR_EFER, &efer);
2405 /* inject #GP if in real mode */
2406 if (ctxt->mode == X86EMUL_MODE_REAL)
2407 return emulate_gp(ctxt, 0);
2410 * Not recognized on AMD in compat mode (but is recognized in legacy
2413 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2414 && !vendor_intel(ctxt))
2415 return emulate_ud(ctxt);
2417 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2418 * Therefore, we inject an #UD.
2420 if (ctxt->mode == X86EMUL_MODE_PROT64)
2421 return emulate_ud(ctxt);
2423 setup_syscalls_segments(ctxt, &cs, &ss);
2425 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2426 switch (ctxt->mode) {
2427 case X86EMUL_MODE_PROT32:
2428 if ((msr_data & 0xfffc) == 0x0)
2429 return emulate_gp(ctxt, 0);
2431 case X86EMUL_MODE_PROT64:
2432 if (msr_data == 0x0)
2433 return emulate_gp(ctxt, 0);
2439 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2440 cs_sel = (u16)msr_data;
2441 cs_sel &= ~SELECTOR_RPL_MASK;
2442 ss_sel = cs_sel + 8;
2443 ss_sel &= ~SELECTOR_RPL_MASK;
2444 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2449 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2450 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2452 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2453 ctxt->_eip = msr_data;
2455 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2456 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2458 return X86EMUL_CONTINUE;
2461 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2463 const struct x86_emulate_ops *ops = ctxt->ops;
2464 struct desc_struct cs, ss;
2467 u16 cs_sel = 0, ss_sel = 0;
2469 /* inject #GP if in real mode or Virtual 8086 mode */
2470 if (ctxt->mode == X86EMUL_MODE_REAL ||
2471 ctxt->mode == X86EMUL_MODE_VM86)
2472 return emulate_gp(ctxt, 0);
2474 setup_syscalls_segments(ctxt, &cs, &ss);
2476 if ((ctxt->rex_prefix & 0x8) != 0x0)
2477 usermode = X86EMUL_MODE_PROT64;
2479 usermode = X86EMUL_MODE_PROT32;
2483 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2485 case X86EMUL_MODE_PROT32:
2486 cs_sel = (u16)(msr_data + 16);
2487 if ((msr_data & 0xfffc) == 0x0)
2488 return emulate_gp(ctxt, 0);
2489 ss_sel = (u16)(msr_data + 24);
2491 case X86EMUL_MODE_PROT64:
2492 cs_sel = (u16)(msr_data + 32);
2493 if (msr_data == 0x0)
2494 return emulate_gp(ctxt, 0);
2495 ss_sel = cs_sel + 8;
2500 cs_sel |= SELECTOR_RPL_MASK;
2501 ss_sel |= SELECTOR_RPL_MASK;
2503 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2504 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2506 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2507 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2509 return X86EMUL_CONTINUE;
2512 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2515 if (ctxt->mode == X86EMUL_MODE_REAL)
2517 if (ctxt->mode == X86EMUL_MODE_VM86)
2519 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2520 return ctxt->ops->cpl(ctxt) > iopl;
2523 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2526 const struct x86_emulate_ops *ops = ctxt->ops;
2527 struct desc_struct tr_seg;
2530 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2531 unsigned mask = (1 << len) - 1;
2534 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2537 if (desc_limit_scaled(&tr_seg) < 103)
2539 base = get_desc_base(&tr_seg);
2540 #ifdef CONFIG_X86_64
2541 base |= ((u64)base3) << 32;
2543 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2544 if (r != X86EMUL_CONTINUE)
2546 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2548 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2549 if (r != X86EMUL_CONTINUE)
2551 if ((perm >> bit_idx) & mask)
2556 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2562 if (emulator_bad_iopl(ctxt))
2563 if (!emulator_io_port_access_allowed(ctxt, port, len))
2566 ctxt->perm_ok = true;
2571 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2572 struct tss_segment_16 *tss)
2574 tss->ip = ctxt->_eip;
2575 tss->flag = ctxt->eflags;
2576 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2577 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2578 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2579 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2580 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2581 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2582 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2583 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2585 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2586 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2587 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2588 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2589 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2592 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2593 struct tss_segment_16 *tss)
2597 ctxt->_eip = tss->ip;
2598 ctxt->eflags = tss->flag | 2;
2599 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2600 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2601 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2602 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2603 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2604 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2605 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2606 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2609 * SDM says that segment selectors are loaded before segment
2612 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2613 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2614 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2615 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2616 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2619 * Now load segment descriptors. If fault happens at this stage
2620 * it is handled in a context of new task
2622 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2626 if (ret != X86EMUL_CONTINUE)
2628 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2635 if (ret != X86EMUL_CONTINUE)
2638 return X86EMUL_CONTINUE;
2641 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2642 u16 tss_selector, u16 old_tss_sel,
2643 ulong old_tss_base, struct desc_struct *new_desc)
2645 const struct x86_emulate_ops *ops = ctxt->ops;
2646 struct tss_segment_16 tss_seg;
2648 u32 new_tss_base = get_desc_base(new_desc);
2650 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2652 if (ret != X86EMUL_CONTINUE)
2653 /* FIXME: need to provide precise fault address */
2656 save_state_to_tss16(ctxt, &tss_seg);
2658 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2660 if (ret != X86EMUL_CONTINUE)
2661 /* FIXME: need to provide precise fault address */
2664 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2666 if (ret != X86EMUL_CONTINUE)
2667 /* FIXME: need to provide precise fault address */
2670 if (old_tss_sel != 0xffff) {
2671 tss_seg.prev_task_link = old_tss_sel;
2673 ret = ops->write_std(ctxt, new_tss_base,
2674 &tss_seg.prev_task_link,
2675 sizeof tss_seg.prev_task_link,
2677 if (ret != X86EMUL_CONTINUE)
2678 /* FIXME: need to provide precise fault address */
2682 return load_state_from_tss16(ctxt, &tss_seg);
2685 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2686 struct tss_segment_32 *tss)
2688 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2689 tss->eip = ctxt->_eip;
2690 tss->eflags = ctxt->eflags;
2691 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2692 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2693 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2694 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2695 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2696 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2697 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2698 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2700 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2701 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2702 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2703 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2704 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2705 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2706 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2709 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2710 struct tss_segment_32 *tss)
2714 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2715 return emulate_gp(ctxt, 0);
2716 ctxt->_eip = tss->eip;
2717 ctxt->eflags = tss->eflags | 2;
2719 /* General purpose registers */
2720 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2721 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2722 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2723 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2724 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2725 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2726 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2727 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2730 * SDM says that segment selectors are loaded before segment
2733 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2734 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2735 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2736 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2737 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2738 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2739 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2742 * If we're switching between Protected Mode and VM86, we need to make
2743 * sure to update the mode before loading the segment descriptors so
2744 * that the selectors are interpreted correctly.
2746 * Need to get rflags to the vcpu struct immediately because it
2747 * influences the CPL which is checked at least when loading the segment
2748 * descriptors and when pushing an error code to the new kernel stack.
2750 * TODO Introduce a separate ctxt->ops->set_cpl callback
2752 if (ctxt->eflags & X86_EFLAGS_VM)
2753 ctxt->mode = X86EMUL_MODE_VM86;
2755 ctxt->mode = X86EMUL_MODE_PROT32;
2757 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2760 * Now load segment descriptors. If fault happenes at this stage
2761 * it is handled in a context of new task
2763 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2764 if (ret != X86EMUL_CONTINUE)
2766 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2767 if (ret != X86EMUL_CONTINUE)
2769 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2770 if (ret != X86EMUL_CONTINUE)
2772 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2776 if (ret != X86EMUL_CONTINUE)
2778 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2779 if (ret != X86EMUL_CONTINUE)
2781 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2782 if (ret != X86EMUL_CONTINUE)
2785 return X86EMUL_CONTINUE;
2788 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2789 u16 tss_selector, u16 old_tss_sel,
2790 ulong old_tss_base, struct desc_struct *new_desc)
2792 const struct x86_emulate_ops *ops = ctxt->ops;
2793 struct tss_segment_32 tss_seg;
2795 u32 new_tss_base = get_desc_base(new_desc);
2797 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2799 if (ret != X86EMUL_CONTINUE)
2800 /* FIXME: need to provide precise fault address */
2803 save_state_to_tss32(ctxt, &tss_seg);
2805 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2807 if (ret != X86EMUL_CONTINUE)
2808 /* FIXME: need to provide precise fault address */
2811 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2813 if (ret != X86EMUL_CONTINUE)
2814 /* FIXME: need to provide precise fault address */
2817 if (old_tss_sel != 0xffff) {
2818 tss_seg.prev_task_link = old_tss_sel;
2820 ret = ops->write_std(ctxt, new_tss_base,
2821 &tss_seg.prev_task_link,
2822 sizeof tss_seg.prev_task_link,
2824 if (ret != X86EMUL_CONTINUE)
2825 /* FIXME: need to provide precise fault address */
2829 return load_state_from_tss32(ctxt, &tss_seg);
2832 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2833 u16 tss_selector, int idt_index, int reason,
2834 bool has_error_code, u32 error_code)
2836 const struct x86_emulate_ops *ops = ctxt->ops;
2837 struct desc_struct curr_tss_desc, next_tss_desc;
2839 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2840 ulong old_tss_base =
2841 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2845 /* FIXME: old_tss_base == ~0 ? */
2847 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2848 if (ret != X86EMUL_CONTINUE)
2850 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2851 if (ret != X86EMUL_CONTINUE)
2854 /* FIXME: check that next_tss_desc is tss */
2857 * Check privileges. The three cases are task switch caused by...
2859 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2860 * 2. Exception/IRQ/iret: No check is performed
2861 * 3. jmp/call to TSS: Check against DPL of the TSS
2863 if (reason == TASK_SWITCH_GATE) {
2864 if (idt_index != -1) {
2865 /* Software interrupts */
2866 struct desc_struct task_gate_desc;
2869 ret = read_interrupt_descriptor(ctxt, idt_index,
2871 if (ret != X86EMUL_CONTINUE)
2874 dpl = task_gate_desc.dpl;
2875 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2876 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2878 } else if (reason != TASK_SWITCH_IRET) {
2879 int dpl = next_tss_desc.dpl;
2880 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2881 return emulate_gp(ctxt, tss_selector);
2885 desc_limit = desc_limit_scaled(&next_tss_desc);
2886 if (!next_tss_desc.p ||
2887 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2888 desc_limit < 0x2b)) {
2889 emulate_ts(ctxt, tss_selector & 0xfffc);
2890 return X86EMUL_PROPAGATE_FAULT;
2893 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2894 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2895 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2898 if (reason == TASK_SWITCH_IRET)
2899 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2901 /* set back link to prev task only if NT bit is set in eflags
2902 note that old_tss_sel is not used after this point */
2903 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2904 old_tss_sel = 0xffff;
2906 if (next_tss_desc.type & 8)
2907 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2908 old_tss_base, &next_tss_desc);
2910 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2911 old_tss_base, &next_tss_desc);
2912 if (ret != X86EMUL_CONTINUE)
2915 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2916 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2918 if (reason != TASK_SWITCH_IRET) {
2919 next_tss_desc.type |= (1 << 1); /* set busy flag */
2920 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2923 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2924 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2926 if (has_error_code) {
2927 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2928 ctxt->lock_prefix = 0;
2929 ctxt->src.val = (unsigned long) error_code;
2930 ret = em_push(ctxt);
2936 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2937 u16 tss_selector, int idt_index, int reason,
2938 bool has_error_code, u32 error_code)
2942 invalidate_registers(ctxt);
2943 ctxt->_eip = ctxt->eip;
2944 ctxt->dst.type = OP_NONE;
2946 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2947 has_error_code, error_code);
2949 if (rc == X86EMUL_CONTINUE) {
2950 ctxt->eip = ctxt->_eip;
2951 writeback_registers(ctxt);
2954 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2957 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2960 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2962 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2963 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2966 static int em_das(struct x86_emulate_ctxt *ctxt)
2969 bool af, cf, old_cf;
2971 cf = ctxt->eflags & X86_EFLAGS_CF;
2977 af = ctxt->eflags & X86_EFLAGS_AF;
2978 if ((al & 0x0f) > 9 || af) {
2980 cf = old_cf | (al >= 250);
2985 if (old_al > 0x99 || old_cf) {
2991 /* Set PF, ZF, SF */
2992 ctxt->src.type = OP_IMM;
2994 ctxt->src.bytes = 1;
2995 fastop(ctxt, em_or);
2996 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2998 ctxt->eflags |= X86_EFLAGS_CF;
3000 ctxt->eflags |= X86_EFLAGS_AF;
3001 return X86EMUL_CONTINUE;
3004 static int em_aam(struct x86_emulate_ctxt *ctxt)
3008 if (ctxt->src.val == 0)
3009 return emulate_de(ctxt);
3011 al = ctxt->dst.val & 0xff;
3012 ah = al / ctxt->src.val;
3013 al %= ctxt->src.val;
3015 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3017 /* Set PF, ZF, SF */
3018 ctxt->src.type = OP_IMM;
3020 ctxt->src.bytes = 1;
3021 fastop(ctxt, em_or);
3023 return X86EMUL_CONTINUE;
3026 static int em_aad(struct x86_emulate_ctxt *ctxt)
3028 u8 al = ctxt->dst.val & 0xff;
3029 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3031 al = (al + (ah * ctxt->src.val)) & 0xff;
3033 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3035 /* Set PF, ZF, SF */
3036 ctxt->src.type = OP_IMM;
3038 ctxt->src.bytes = 1;
3039 fastop(ctxt, em_or);
3041 return X86EMUL_CONTINUE;
3044 static int em_call(struct x86_emulate_ctxt *ctxt)
3046 long rel = ctxt->src.val;
3048 ctxt->src.val = (unsigned long)ctxt->_eip;
3050 return em_push(ctxt);
3053 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3059 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3060 old_eip = ctxt->_eip;
3062 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3063 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3064 return X86EMUL_CONTINUE;
3067 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3069 ctxt->src.val = old_cs;
3071 if (rc != X86EMUL_CONTINUE)
3074 ctxt->src.val = old_eip;
3075 return em_push(ctxt);
3078 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3082 ctxt->dst.type = OP_REG;
3083 ctxt->dst.addr.reg = &ctxt->_eip;
3084 ctxt->dst.bytes = ctxt->op_bytes;
3085 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3086 if (rc != X86EMUL_CONTINUE)
3088 rsp_increment(ctxt, ctxt->src.val);
3089 return X86EMUL_CONTINUE;
3092 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3094 /* Write back the register source. */
3095 ctxt->src.val = ctxt->dst.val;
3096 write_register_operand(&ctxt->src);
3098 /* Write back the memory destination with implicit LOCK prefix. */
3099 ctxt->dst.val = ctxt->src.orig_val;
3100 ctxt->lock_prefix = 1;
3101 return X86EMUL_CONTINUE;
3104 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3106 ctxt->dst.val = ctxt->src2.val;
3107 return fastop(ctxt, em_imul);
3110 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3112 ctxt->dst.type = OP_REG;
3113 ctxt->dst.bytes = ctxt->src.bytes;
3114 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3115 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3117 return X86EMUL_CONTINUE;
3120 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3124 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3125 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3126 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3127 return X86EMUL_CONTINUE;
3130 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3134 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3135 return emulate_gp(ctxt, 0);
3136 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3137 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3138 return X86EMUL_CONTINUE;
3141 static int em_mov(struct x86_emulate_ctxt *ctxt)
3143 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3144 return X86EMUL_CONTINUE;
3147 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3149 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3150 return emulate_gp(ctxt, 0);
3152 /* Disable writeback. */
3153 ctxt->dst.type = OP_NONE;
3154 return X86EMUL_CONTINUE;
3157 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3161 if (ctxt->mode == X86EMUL_MODE_PROT64)
3162 val = ctxt->src.val & ~0ULL;
3164 val = ctxt->src.val & ~0U;
3166 /* #UD condition is already handled. */
3167 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3168 return emulate_gp(ctxt, 0);
3170 /* Disable writeback. */
3171 ctxt->dst.type = OP_NONE;
3172 return X86EMUL_CONTINUE;
3175 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3179 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3180 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3181 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3182 return emulate_gp(ctxt, 0);
3184 return X86EMUL_CONTINUE;
3187 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3191 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3192 return emulate_gp(ctxt, 0);
3194 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3195 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3196 return X86EMUL_CONTINUE;
3199 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3201 if (ctxt->modrm_reg > VCPU_SREG_GS)
3202 return emulate_ud(ctxt);
3204 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3205 return X86EMUL_CONTINUE;
3208 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3210 u16 sel = ctxt->src.val;
3212 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3213 return emulate_ud(ctxt);
3215 if (ctxt->modrm_reg == VCPU_SREG_SS)
3216 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3218 /* Disable writeback. */
3219 ctxt->dst.type = OP_NONE;
3220 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3223 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3225 u16 sel = ctxt->src.val;
3227 /* Disable writeback. */
3228 ctxt->dst.type = OP_NONE;
3229 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3232 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3234 u16 sel = ctxt->src.val;
3236 /* Disable writeback. */
3237 ctxt->dst.type = OP_NONE;
3238 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3241 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3246 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3247 if (rc == X86EMUL_CONTINUE)
3248 ctxt->ops->invlpg(ctxt, linear);
3249 /* Disable writeback. */
3250 ctxt->dst.type = OP_NONE;
3251 return X86EMUL_CONTINUE;
3254 static int em_clts(struct x86_emulate_ctxt *ctxt)
3258 cr0 = ctxt->ops->get_cr(ctxt, 0);
3260 ctxt->ops->set_cr(ctxt, 0, cr0);
3261 return X86EMUL_CONTINUE;
3264 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3268 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3269 return X86EMUL_UNHANDLEABLE;
3271 rc = ctxt->ops->fix_hypercall(ctxt);
3272 if (rc != X86EMUL_CONTINUE)
3275 /* Let the processor re-execute the fixed hypercall */
3276 ctxt->_eip = ctxt->eip;
3277 /* Disable writeback. */
3278 ctxt->dst.type = OP_NONE;
3279 return X86EMUL_CONTINUE;
3282 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3283 void (*get)(struct x86_emulate_ctxt *ctxt,
3284 struct desc_ptr *ptr))
3286 struct desc_ptr desc_ptr;
3288 if (ctxt->mode == X86EMUL_MODE_PROT64)
3290 get(ctxt, &desc_ptr);
3291 if (ctxt->op_bytes == 2) {
3293 desc_ptr.address &= 0x00ffffff;
3295 /* Disable writeback. */
3296 ctxt->dst.type = OP_NONE;
3297 return segmented_write(ctxt, ctxt->dst.addr.mem,
3298 &desc_ptr, 2 + ctxt->op_bytes);
3301 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3303 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3306 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3308 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3311 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3313 struct desc_ptr desc_ptr;
3316 if (ctxt->mode == X86EMUL_MODE_PROT64)
3318 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3319 &desc_ptr.size, &desc_ptr.address,
3321 if (rc != X86EMUL_CONTINUE)
3323 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3324 /* Disable writeback. */
3325 ctxt->dst.type = OP_NONE;
3326 return X86EMUL_CONTINUE;
3329 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3333 rc = ctxt->ops->fix_hypercall(ctxt);
3335 /* Disable writeback. */
3336 ctxt->dst.type = OP_NONE;
3340 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3342 struct desc_ptr desc_ptr;
3345 if (ctxt->mode == X86EMUL_MODE_PROT64)
3347 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3348 &desc_ptr.size, &desc_ptr.address,
3350 if (rc != X86EMUL_CONTINUE)
3352 ctxt->ops->set_idt(ctxt, &desc_ptr);
3353 /* Disable writeback. */
3354 ctxt->dst.type = OP_NONE;
3355 return X86EMUL_CONTINUE;
3358 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3360 ctxt->dst.bytes = 2;
3361 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3362 return X86EMUL_CONTINUE;
3365 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3367 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3368 | (ctxt->src.val & 0x0f));
3369 ctxt->dst.type = OP_NONE;
3370 return X86EMUL_CONTINUE;
3373 static int em_loop(struct x86_emulate_ctxt *ctxt)
3375 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3376 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3377 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3378 jmp_rel(ctxt, ctxt->src.val);
3380 return X86EMUL_CONTINUE;
3383 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3385 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3386 jmp_rel(ctxt, ctxt->src.val);
3388 return X86EMUL_CONTINUE;
3391 static int em_in(struct x86_emulate_ctxt *ctxt)
3393 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3395 return X86EMUL_IO_NEEDED;
3397 return X86EMUL_CONTINUE;
3400 static int em_out(struct x86_emulate_ctxt *ctxt)
3402 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3406 return X86EMUL_CONTINUE;
3409 static int em_cli(struct x86_emulate_ctxt *ctxt)
3411 if (emulator_bad_iopl(ctxt))
3412 return emulate_gp(ctxt, 0);
3414 ctxt->eflags &= ~X86_EFLAGS_IF;
3415 return X86EMUL_CONTINUE;
3418 static int em_sti(struct x86_emulate_ctxt *ctxt)
3420 if (emulator_bad_iopl(ctxt))
3421 return emulate_gp(ctxt, 0);
3423 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3424 ctxt->eflags |= X86_EFLAGS_IF;
3425 return X86EMUL_CONTINUE;
3428 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3430 u32 eax, ebx, ecx, edx;
3432 eax = reg_read(ctxt, VCPU_REGS_RAX);
3433 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3434 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3435 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3436 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3437 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3438 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3439 return X86EMUL_CONTINUE;
3442 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3444 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3445 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3446 return X86EMUL_CONTINUE;
3449 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3451 switch (ctxt->op_bytes) {
3452 #ifdef CONFIG_X86_64
3454 asm("bswap %0" : "+r"(ctxt->dst.val));
3458 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3461 return X86EMUL_CONTINUE;
3464 static bool valid_cr(int nr)
3476 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3478 if (!valid_cr(ctxt->modrm_reg))
3479 return emulate_ud(ctxt);
3481 return X86EMUL_CONTINUE;
3484 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3486 u64 new_val = ctxt->src.val64;
3487 int cr = ctxt->modrm_reg;
3490 static u64 cr_reserved_bits[] = {
3491 0xffffffff00000000ULL,
3492 0, 0, 0, /* CR3 checked later */
3499 return emulate_ud(ctxt);
3501 if (new_val & cr_reserved_bits[cr])
3502 return emulate_gp(ctxt, 0);
3507 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3508 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3509 return emulate_gp(ctxt, 0);
3511 cr4 = ctxt->ops->get_cr(ctxt, 4);
3512 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3514 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3515 !(cr4 & X86_CR4_PAE))
3516 return emulate_gp(ctxt, 0);
3523 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3524 if (efer & EFER_LMA)
3525 rsvd = CR3_L_MODE_RESERVED_BITS;
3526 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3527 rsvd = CR3_PAE_RESERVED_BITS;
3528 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3529 rsvd = CR3_NONPAE_RESERVED_BITS;
3532 return emulate_gp(ctxt, 0);
3537 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3539 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3540 return emulate_gp(ctxt, 0);
3546 return X86EMUL_CONTINUE;
3549 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3553 ctxt->ops->get_dr(ctxt, 7, &dr7);
3555 /* Check if DR7.Global_Enable is set */
3556 return dr7 & (1 << 13);
3559 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3561 int dr = ctxt->modrm_reg;
3565 return emulate_ud(ctxt);
3567 cr4 = ctxt->ops->get_cr(ctxt, 4);
3568 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3569 return emulate_ud(ctxt);
3571 if (check_dr7_gd(ctxt))
3572 return emulate_db(ctxt);
3574 return X86EMUL_CONTINUE;
3577 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3579 u64 new_val = ctxt->src.val64;
3580 int dr = ctxt->modrm_reg;
3582 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3583 return emulate_gp(ctxt, 0);
3585 return check_dr_read(ctxt);
3588 static int check_svme(struct x86_emulate_ctxt *ctxt)
3592 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3594 if (!(efer & EFER_SVME))
3595 return emulate_ud(ctxt);
3597 return X86EMUL_CONTINUE;
3600 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3602 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3604 /* Valid physical address? */
3605 if (rax & 0xffff000000000000ULL)
3606 return emulate_gp(ctxt, 0);
3608 return check_svme(ctxt);
3611 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3613 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3615 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3616 return emulate_ud(ctxt);
3618 return X86EMUL_CONTINUE;
3621 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3623 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3624 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3626 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3628 return emulate_gp(ctxt, 0);
3630 return X86EMUL_CONTINUE;
3633 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3635 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3636 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3637 return emulate_gp(ctxt, 0);
3639 return X86EMUL_CONTINUE;
3642 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3644 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3645 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3646 return emulate_gp(ctxt, 0);
3648 return X86EMUL_CONTINUE;
3651 #define D(_y) { .flags = (_y) }
3652 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3653 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3654 .check_perm = (_p) }
3655 #define N D(NotImpl)
3656 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3657 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3658 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3659 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3660 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3661 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3662 #define II(_f, _e, _i) \
3663 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3664 #define IIP(_f, _e, _i, _p) \
3665 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3666 .check_perm = (_p) }
3667 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3669 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3670 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3671 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3672 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3673 #define I2bvIP(_f, _e, _i, _p) \
3674 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3676 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3677 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3678 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3680 static const struct opcode group7_rm1[] = {
3681 DI(SrcNone | Priv, monitor),
3682 DI(SrcNone | Priv, mwait),
3686 static const struct opcode group7_rm3[] = {
3687 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3688 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3689 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3690 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3691 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3692 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3693 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3694 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3697 static const struct opcode group7_rm7[] = {
3699 DIP(SrcNone, rdtscp, check_rdtsc),
3703 static const struct opcode group1[] = {
3705 F(Lock | PageTable, em_or),
3708 F(Lock | PageTable, em_and),
3714 static const struct opcode group1A[] = {
3715 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3718 static const struct opcode group2[] = {
3719 F(DstMem | ModRM, em_rol),
3720 F(DstMem | ModRM, em_ror),
3721 F(DstMem | ModRM, em_rcl),
3722 F(DstMem | ModRM, em_rcr),
3723 F(DstMem | ModRM, em_shl),
3724 F(DstMem | ModRM, em_shr),
3725 F(DstMem | ModRM, em_shl),
3726 F(DstMem | ModRM, em_sar),
3729 static const struct opcode group3[] = {
3730 F(DstMem | SrcImm | NoWrite, em_test),
3731 F(DstMem | SrcImm | NoWrite, em_test),
3732 F(DstMem | SrcNone | Lock, em_not),
3733 F(DstMem | SrcNone | Lock, em_neg),
3734 I(SrcMem, em_mul_ex),
3735 I(SrcMem, em_imul_ex),
3736 I(SrcMem, em_div_ex),
3737 I(SrcMem, em_idiv_ex),
3740 static const struct opcode group4[] = {
3741 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3742 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3746 static const struct opcode group5[] = {
3747 F(DstMem | SrcNone | Lock, em_inc),
3748 F(DstMem | SrcNone | Lock, em_dec),
3749 I(SrcMem | Stack, em_grp45),
3750 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3751 I(SrcMem | Stack, em_grp45),
3752 I(SrcMemFAddr | ImplicitOps, em_grp45),
3753 I(SrcMem | Stack, em_grp45), D(Undefined),
3756 static const struct opcode group6[] = {
3759 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3760 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3764 static const struct group_dual group7 = { {
3765 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3766 II(Mov | DstMem | Priv, em_sidt, sidt),
3767 II(SrcMem | Priv, em_lgdt, lgdt),
3768 II(SrcMem | Priv, em_lidt, lidt),
3769 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3770 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3771 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3773 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3775 N, EXT(0, group7_rm3),
3776 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3777 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3781 static const struct opcode group8[] = {
3783 F(DstMem | SrcImmByte | NoWrite, em_bt),
3784 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3785 F(DstMem | SrcImmByte | Lock, em_btr),
3786 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3789 static const struct group_dual group9 = { {
3790 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3792 N, N, N, N, N, N, N, N,
3795 static const struct opcode group11[] = {
3796 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3800 static const struct gprefix pfx_0f_6f_0f_7f = {
3801 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3804 static const struct gprefix pfx_vmovntpx = {
3805 I(0, em_mov), N, N, N,
3808 static const struct escape escape_d9 = { {
3809 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3812 N, N, N, N, N, N, N, N,
3814 N, N, N, N, N, N, N, N,
3816 N, N, N, N, N, N, N, N,
3818 N, N, N, N, N, N, N, N,
3820 N, N, N, N, N, N, N, N,
3822 N, N, N, N, N, N, N, N,
3824 N, N, N, N, N, N, N, N,
3826 N, N, N, N, N, N, N, N,
3829 static const struct escape escape_db = { {
3830 N, N, N, N, N, N, N, N,
3833 N, N, N, N, N, N, N, N,
3835 N, N, N, N, N, N, N, N,
3837 N, N, N, N, N, N, N, N,
3839 N, N, N, N, N, N, N, N,
3841 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3843 N, N, N, N, N, N, N, N,
3845 N, N, N, N, N, N, N, N,
3847 N, N, N, N, N, N, N, N,
3850 static const struct escape escape_dd = { {
3851 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3854 N, N, N, N, N, N, N, N,
3856 N, N, N, N, N, N, N, N,
3858 N, N, N, N, N, N, N, N,
3860 N, N, N, N, N, N, N, N,
3862 N, N, N, N, N, N, N, N,
3864 N, N, N, N, N, N, N, N,
3866 N, N, N, N, N, N, N, N,
3868 N, N, N, N, N, N, N, N,
3871 static const struct opcode opcode_table[256] = {
3873 F6ALU(Lock, em_add),
3874 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3875 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3877 F6ALU(Lock | PageTable, em_or),
3878 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3881 F6ALU(Lock, em_adc),
3882 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3883 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3885 F6ALU(Lock, em_sbb),
3886 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3887 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3889 F6ALU(Lock | PageTable, em_and), N, N,
3891 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3893 F6ALU(Lock, em_xor), N, N,
3895 F6ALU(NoWrite, em_cmp), N, N,
3897 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3899 X8(I(SrcReg | Stack, em_push)),
3901 X8(I(DstReg | Stack, em_pop)),
3903 I(ImplicitOps | Stack | No64, em_pusha),
3904 I(ImplicitOps | Stack | No64, em_popa),
3905 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3908 I(SrcImm | Mov | Stack, em_push),
3909 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3910 I(SrcImmByte | Mov | Stack, em_push),
3911 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3912 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3913 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3917 G(ByteOp | DstMem | SrcImm, group1),
3918 G(DstMem | SrcImm, group1),
3919 G(ByteOp | DstMem | SrcImm | No64, group1),
3920 G(DstMem | SrcImmByte, group1),
3921 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3922 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3924 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3925 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3926 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3927 D(ModRM | SrcMem | NoAccess | DstReg),
3928 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3931 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3933 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3934 I(SrcImmFAddr | No64, em_call_far), N,
3935 II(ImplicitOps | Stack, em_pushf, pushf),
3936 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3938 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3939 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3940 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3941 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3943 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3944 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3945 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3946 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3948 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3950 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3952 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3953 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3954 I(ImplicitOps | Stack, em_ret),
3955 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3956 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3957 G(ByteOp, group11), G(0, group11),
3959 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3960 N, I(ImplicitOps | Stack, em_ret_far),
3961 D(ImplicitOps), DI(SrcImmByte, intn),
3962 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3964 G(Src2One | ByteOp, group2), G(Src2One, group2),
3965 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3966 I(DstAcc | SrcImmUByte | No64, em_aam),
3967 I(DstAcc | SrcImmUByte | No64, em_aad),
3968 F(DstAcc | ByteOp | No64, em_salc),
3969 I(DstAcc | SrcXLat | ByteOp, em_mov),
3971 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3973 X3(I(SrcImmByte, em_loop)),
3974 I(SrcImmByte, em_jcxz),
3975 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3976 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3978 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3979 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3980 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3981 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3983 N, DI(ImplicitOps, icebp), N, N,
3984 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3985 G(ByteOp, group3), G(0, group3),
3987 D(ImplicitOps), D(ImplicitOps),
3988 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3989 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3992 static const struct opcode twobyte_table[256] = {
3994 G(0, group6), GD(0, &group7), N, N,
3995 N, I(ImplicitOps | VendorSpecific, em_syscall),
3996 II(ImplicitOps | Priv, em_clts, clts), N,
3997 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3998 N, D(ImplicitOps | ModRM), N, N,
4000 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
4002 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4003 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4004 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4005 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4007 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4010 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4011 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4012 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4013 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4014 I(ImplicitOps | VendorSpecific, em_sysenter),
4015 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4017 N, N, N, N, N, N, N, N,
4019 X16(D(DstReg | SrcMem | ModRM | Mov)),
4021 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4026 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4031 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4035 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4037 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4038 II(ImplicitOps, em_cpuid, cpuid),
4039 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4040 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4041 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4043 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4044 DI(ImplicitOps, rsm),
4045 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4046 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4047 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4048 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4050 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4051 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4052 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4053 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4054 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4055 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4059 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4060 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4061 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4063 D2bv(DstMem | SrcReg | ModRM | Lock),
4064 N, D(DstMem | SrcReg | ModRM | Mov),
4065 N, N, N, GD(0, &group9),
4067 X8(I(DstReg, em_bswap)),
4069 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4071 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4073 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4090 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4094 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4100 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4101 unsigned size, bool sign_extension)
4103 int rc = X86EMUL_CONTINUE;
4107 op->addr.mem.ea = ctxt->_eip;
4108 /* NB. Immediates are sign-extended as necessary. */
4109 switch (op->bytes) {
4111 op->val = insn_fetch(s8, ctxt);
4114 op->val = insn_fetch(s16, ctxt);
4117 op->val = insn_fetch(s32, ctxt);
4120 op->val = insn_fetch(s64, ctxt);
4123 if (!sign_extension) {
4124 switch (op->bytes) {
4132 op->val &= 0xffffffff;
4140 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4143 int rc = X86EMUL_CONTINUE;
4147 decode_register_operand(ctxt, op);
4150 rc = decode_imm(ctxt, op, 1, false);
4153 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4157 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4158 fetch_bit_operand(ctxt);
4159 op->orig_val = op->val;
4162 ctxt->memop.bytes = 8;
4166 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4167 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4168 fetch_register_operand(op);
4169 op->orig_val = op->val;
4173 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4175 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4176 op->addr.mem.seg = VCPU_SREG_ES;
4183 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4184 fetch_register_operand(op);
4188 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4191 rc = decode_imm(ctxt, op, 1, true);
4198 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4201 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4204 ctxt->memop.bytes = 1;
4205 if (ctxt->memop.type == OP_REG) {
4206 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4207 fetch_register_operand(&ctxt->memop);
4211 ctxt->memop.bytes = 2;
4214 ctxt->memop.bytes = 4;
4217 rc = decode_imm(ctxt, op, 2, false);
4220 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4224 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4226 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4227 op->addr.mem.seg = seg_override(ctxt);
4233 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4235 register_address(ctxt,
4236 reg_read(ctxt, VCPU_REGS_RBX) +
4237 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4238 op->addr.mem.seg = seg_override(ctxt);
4243 op->addr.mem.ea = ctxt->_eip;
4244 op->bytes = ctxt->op_bytes + 2;
4245 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4248 ctxt->memop.bytes = ctxt->op_bytes + 2;
4251 op->val = VCPU_SREG_ES;
4254 op->val = VCPU_SREG_CS;
4257 op->val = VCPU_SREG_SS;
4260 op->val = VCPU_SREG_DS;
4263 op->val = VCPU_SREG_FS;
4266 op->val = VCPU_SREG_GS;
4269 /* Special instructions do their own operand decoding. */
4271 op->type = OP_NONE; /* Disable writeback. */
4279 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4281 int rc = X86EMUL_CONTINUE;
4282 int mode = ctxt->mode;
4283 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4284 bool op_prefix = false;
4285 struct opcode opcode;
4287 ctxt->memop.type = OP_NONE;
4288 ctxt->memopp = NULL;
4289 ctxt->_eip = ctxt->eip;
4290 ctxt->fetch.start = ctxt->_eip;
4291 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4293 memcpy(ctxt->fetch.data, insn, insn_len);
4296 case X86EMUL_MODE_REAL:
4297 case X86EMUL_MODE_VM86:
4298 case X86EMUL_MODE_PROT16:
4299 def_op_bytes = def_ad_bytes = 2;
4301 case X86EMUL_MODE_PROT32:
4302 def_op_bytes = def_ad_bytes = 4;
4304 #ifdef CONFIG_X86_64
4305 case X86EMUL_MODE_PROT64:
4311 return EMULATION_FAILED;
4314 ctxt->op_bytes = def_op_bytes;
4315 ctxt->ad_bytes = def_ad_bytes;
4317 /* Legacy prefixes. */
4319 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4320 case 0x66: /* operand-size override */
4322 /* switch between 2/4 bytes */
4323 ctxt->op_bytes = def_op_bytes ^ 6;
4325 case 0x67: /* address-size override */
4326 if (mode == X86EMUL_MODE_PROT64)
4327 /* switch between 4/8 bytes */
4328 ctxt->ad_bytes = def_ad_bytes ^ 12;
4330 /* switch between 2/4 bytes */
4331 ctxt->ad_bytes = def_ad_bytes ^ 6;
4333 case 0x26: /* ES override */
4334 case 0x2e: /* CS override */
4335 case 0x36: /* SS override */
4336 case 0x3e: /* DS override */
4337 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4339 case 0x64: /* FS override */
4340 case 0x65: /* GS override */
4341 set_seg_override(ctxt, ctxt->b & 7);
4343 case 0x40 ... 0x4f: /* REX */
4344 if (mode != X86EMUL_MODE_PROT64)
4346 ctxt->rex_prefix = ctxt->b;
4348 case 0xf0: /* LOCK */
4349 ctxt->lock_prefix = 1;
4351 case 0xf2: /* REPNE/REPNZ */
4352 case 0xf3: /* REP/REPE/REPZ */
4353 ctxt->rep_prefix = ctxt->b;
4359 /* Any legacy prefix after a REX prefix nullifies its effect. */
4361 ctxt->rex_prefix = 0;
4367 if (ctxt->rex_prefix & 8)
4368 ctxt->op_bytes = 8; /* REX.W */
4370 /* Opcode byte(s). */
4371 opcode = opcode_table[ctxt->b];
4372 /* Two-byte opcode? */
4373 if (ctxt->b == 0x0f) {
4375 ctxt->b = insn_fetch(u8, ctxt);
4376 opcode = twobyte_table[ctxt->b];
4378 ctxt->d = opcode.flags;
4380 if (ctxt->d & ModRM)
4381 ctxt->modrm = insn_fetch(u8, ctxt);
4383 while (ctxt->d & GroupMask) {
4384 switch (ctxt->d & GroupMask) {
4386 goffset = (ctxt->modrm >> 3) & 7;
4387 opcode = opcode.u.group[goffset];
4390 goffset = (ctxt->modrm >> 3) & 7;
4391 if ((ctxt->modrm >> 6) == 3)
4392 opcode = opcode.u.gdual->mod3[goffset];
4394 opcode = opcode.u.gdual->mod012[goffset];
4397 goffset = ctxt->modrm & 7;
4398 opcode = opcode.u.group[goffset];
4401 if (ctxt->rep_prefix && op_prefix)
4402 return EMULATION_FAILED;
4403 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4404 switch (simd_prefix) {
4405 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4406 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4407 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4408 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4412 if (ctxt->modrm > 0xbf)
4413 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4415 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4418 return EMULATION_FAILED;
4421 ctxt->d &= ~(u64)GroupMask;
4422 ctxt->d |= opcode.flags;
4425 ctxt->execute = opcode.u.execute;
4426 ctxt->check_perm = opcode.check_perm;
4427 ctxt->intercept = opcode.intercept;
4430 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4431 return EMULATION_FAILED;
4433 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4434 return EMULATION_FAILED;
4436 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4439 if (ctxt->d & Op3264) {
4440 if (mode == X86EMUL_MODE_PROT64)
4447 ctxt->op_bytes = 16;
4448 else if (ctxt->d & Mmx)
4451 /* ModRM and SIB bytes. */
4452 if (ctxt->d & ModRM) {
4453 rc = decode_modrm(ctxt, &ctxt->memop);
4454 if (!ctxt->has_seg_override)
4455 set_seg_override(ctxt, ctxt->modrm_seg);
4456 } else if (ctxt->d & MemAbs)
4457 rc = decode_abs(ctxt, &ctxt->memop);
4458 if (rc != X86EMUL_CONTINUE)
4461 if (!ctxt->has_seg_override)
4462 set_seg_override(ctxt, VCPU_SREG_DS);
4464 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4466 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4467 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4470 * Decode and fetch the source operand: register, memory
4473 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4474 if (rc != X86EMUL_CONTINUE)
4478 * Decode and fetch the second source operand: register, memory
4481 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4482 if (rc != X86EMUL_CONTINUE)
4485 /* Decode and fetch the destination operand: register or memory. */
4486 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4489 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4490 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4492 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4495 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4497 return ctxt->d & PageTable;
4500 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4502 /* The second termination condition only applies for REPE
4503 * and REPNE. Test if the repeat string operation prefix is
4504 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4505 * corresponding termination condition according to:
4506 * - if REPE/REPZ and ZF = 0 then done
4507 * - if REPNE/REPNZ and ZF = 1 then done
4509 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4510 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4511 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4512 ((ctxt->eflags & EFLG_ZF) == 0))
4513 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4514 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4520 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4524 ctxt->ops->get_fpu(ctxt);
4525 asm volatile("1: fwait \n\t"
4527 ".pushsection .fixup,\"ax\" \n\t"
4529 "movb $1, %[fault] \n\t"
4532 _ASM_EXTABLE(1b, 3b)
4533 : [fault]"+qm"(fault));
4534 ctxt->ops->put_fpu(ctxt);
4536 if (unlikely(fault))
4537 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4539 return X86EMUL_CONTINUE;
4542 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4545 if (op->type == OP_MM)
4546 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4549 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4551 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4552 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4553 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4554 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4555 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4556 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4557 return X86EMUL_CONTINUE;
4560 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4562 const struct x86_emulate_ops *ops = ctxt->ops;
4563 int rc = X86EMUL_CONTINUE;
4564 int saved_dst_type = ctxt->dst.type;
4566 ctxt->mem_read.pos = 0;
4568 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4569 (ctxt->d & Undefined)) {
4570 rc = emulate_ud(ctxt);
4574 /* LOCK prefix is allowed only with some instructions */
4575 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4576 rc = emulate_ud(ctxt);
4580 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4581 rc = emulate_ud(ctxt);
4585 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4586 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4587 rc = emulate_ud(ctxt);
4591 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4592 rc = emulate_nm(ctxt);
4596 if (ctxt->d & Mmx) {
4597 rc = flush_pending_x87_faults(ctxt);
4598 if (rc != X86EMUL_CONTINUE)
4601 * Now that we know the fpu is exception safe, we can fetch
4604 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4605 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4606 if (!(ctxt->d & Mov))
4607 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4610 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4611 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4612 X86_ICPT_PRE_EXCEPT);
4613 if (rc != X86EMUL_CONTINUE)
4617 /* Privileged instruction can be executed only in CPL=0 */
4618 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4619 rc = emulate_gp(ctxt, 0);
4623 /* Instruction can only be executed in protected mode */
4624 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4625 rc = emulate_ud(ctxt);
4629 /* Do instruction specific permission checks */
4630 if (ctxt->check_perm) {
4631 rc = ctxt->check_perm(ctxt);
4632 if (rc != X86EMUL_CONTINUE)
4636 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4637 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4638 X86_ICPT_POST_EXCEPT);
4639 if (rc != X86EMUL_CONTINUE)
4643 if (ctxt->rep_prefix && (ctxt->d & String)) {
4644 /* All REP prefixes have the same first termination condition */
4645 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4646 ctxt->eip = ctxt->_eip;
4651 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4652 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4653 ctxt->src.valptr, ctxt->src.bytes);
4654 if (rc != X86EMUL_CONTINUE)
4656 ctxt->src.orig_val64 = ctxt->src.val64;
4659 if (ctxt->src2.type == OP_MEM) {
4660 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4661 &ctxt->src2.val, ctxt->src2.bytes);
4662 if (rc != X86EMUL_CONTINUE)
4666 if ((ctxt->d & DstMask) == ImplicitOps)
4670 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4671 /* optimisation - avoid slow emulated read if Mov */
4672 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4673 &ctxt->dst.val, ctxt->dst.bytes);
4674 if (rc != X86EMUL_CONTINUE)
4677 ctxt->dst.orig_val = ctxt->dst.val;
4681 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4682 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4683 X86_ICPT_POST_MEMACCESS);
4684 if (rc != X86EMUL_CONTINUE)
4688 if (ctxt->execute) {
4689 if (ctxt->d & Fastop) {
4690 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4691 rc = fastop(ctxt, fop);
4692 if (rc != X86EMUL_CONTINUE)
4696 rc = ctxt->execute(ctxt);
4697 if (rc != X86EMUL_CONTINUE)
4706 case 0x63: /* movsxd */
4707 if (ctxt->mode != X86EMUL_MODE_PROT64)
4708 goto cannot_emulate;
4709 ctxt->dst.val = (s32) ctxt->src.val;
4711 case 0x70 ... 0x7f: /* jcc (short) */
4712 if (test_cc(ctxt->b, ctxt->eflags))
4713 jmp_rel(ctxt, ctxt->src.val);
4715 case 0x8d: /* lea r16/r32, m */
4716 ctxt->dst.val = ctxt->src.addr.mem.ea;
4718 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4719 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4723 case 0x98: /* cbw/cwde/cdqe */
4724 switch (ctxt->op_bytes) {
4725 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4726 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4727 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4730 case 0xcc: /* int3 */
4731 rc = emulate_int(ctxt, 3);
4733 case 0xcd: /* int n */
4734 rc = emulate_int(ctxt, ctxt->src.val);
4736 case 0xce: /* into */
4737 if (ctxt->eflags & EFLG_OF)
4738 rc = emulate_int(ctxt, 4);
4740 case 0xe9: /* jmp rel */
4741 case 0xeb: /* jmp rel short */
4742 jmp_rel(ctxt, ctxt->src.val);
4743 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4745 case 0xf4: /* hlt */
4746 ctxt->ops->halt(ctxt);
4748 case 0xf5: /* cmc */
4749 /* complement carry flag from eflags reg */
4750 ctxt->eflags ^= EFLG_CF;
4752 case 0xf8: /* clc */
4753 ctxt->eflags &= ~EFLG_CF;
4755 case 0xf9: /* stc */
4756 ctxt->eflags |= EFLG_CF;
4758 case 0xfc: /* cld */
4759 ctxt->eflags &= ~EFLG_DF;
4761 case 0xfd: /* std */
4762 ctxt->eflags |= EFLG_DF;
4765 goto cannot_emulate;
4768 if (rc != X86EMUL_CONTINUE)
4772 rc = writeback(ctxt);
4773 if (rc != X86EMUL_CONTINUE)
4777 * restore dst type in case the decoding will be reused
4778 * (happens for string instruction )
4780 ctxt->dst.type = saved_dst_type;
4782 if ((ctxt->d & SrcMask) == SrcSI)
4783 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4785 if ((ctxt->d & DstMask) == DstDI)
4786 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4788 if (ctxt->rep_prefix && (ctxt->d & String)) {
4790 struct read_cache *r = &ctxt->io_read;
4791 if ((ctxt->d & SrcMask) == SrcSI)
4792 count = ctxt->src.count;
4794 count = ctxt->dst.count;
4795 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4798 if (!string_insn_completed(ctxt)) {
4800 * Re-enter guest when pio read ahead buffer is empty
4801 * or, if it is not used, after each 1024 iteration.
4803 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4804 (r->end == 0 || r->end != r->pos)) {
4806 * Reset read cache. Usually happens before
4807 * decode, but since instruction is restarted
4808 * we have to do it here.
4810 ctxt->mem_read.end = 0;
4811 writeback_registers(ctxt);
4812 return EMULATION_RESTART;
4814 goto done; /* skip rip writeback */
4818 ctxt->eip = ctxt->_eip;
4821 if (rc == X86EMUL_PROPAGATE_FAULT)
4822 ctxt->have_exception = true;
4823 if (rc == X86EMUL_INTERCEPTED)
4824 return EMULATION_INTERCEPTED;
4826 if (rc == X86EMUL_CONTINUE)
4827 writeback_registers(ctxt);
4829 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4833 case 0x09: /* wbinvd */
4834 (ctxt->ops->wbinvd)(ctxt);
4836 case 0x08: /* invd */
4837 case 0x0d: /* GrpP (prefetch) */
4838 case 0x18: /* Grp16 (prefetch/nop) */
4840 case 0x20: /* mov cr, reg */
4841 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4843 case 0x21: /* mov from dr to reg */
4844 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4846 case 0x40 ... 0x4f: /* cmov */
4847 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4848 if (!test_cc(ctxt->b, ctxt->eflags))
4849 ctxt->dst.type = OP_NONE; /* no writeback */
4851 case 0x80 ... 0x8f: /* jnz rel, etc*/
4852 if (test_cc(ctxt->b, ctxt->eflags))
4853 jmp_rel(ctxt, ctxt->src.val);
4855 case 0x90 ... 0x9f: /* setcc r/m8 */
4856 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4858 case 0xae: /* clflush */
4860 case 0xb6 ... 0xb7: /* movzx */
4861 ctxt->dst.bytes = ctxt->op_bytes;
4862 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4863 : (u16) ctxt->src.val;
4865 case 0xbe ... 0xbf: /* movsx */
4866 ctxt->dst.bytes = ctxt->op_bytes;
4867 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4868 (s16) ctxt->src.val;
4870 case 0xc0 ... 0xc1: /* xadd */
4871 fastop(ctxt, em_add);
4872 /* Write back the register source. */
4873 ctxt->src.val = ctxt->dst.orig_val;
4874 write_register_operand(&ctxt->src);
4876 case 0xc3: /* movnti */
4877 ctxt->dst.bytes = ctxt->op_bytes;
4878 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4879 (u64) ctxt->src.val;
4882 goto cannot_emulate;
4885 if (rc != X86EMUL_CONTINUE)
4891 return EMULATION_FAILED;
4894 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4896 invalidate_registers(ctxt);
4899 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4901 writeback_registers(ctxt);