1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpBits 5 /* Width of operand field */
66 #define OpMask ((1ull << OpBits) - 1)
69 * Opcode effective-address decode tables.
70 * Note that we only emulate instructions that have at least one memory
71 * operand (excluding implicit stack references). We assume that stack
72 * references and instruction fetches will never occur in special memory
73 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
77 /* Operand sizes: 8-bit operands or specified/overridden size. */
78 #define ByteOp (1<<0) /* 8-bit operands. */
79 /* Destination operand type. */
81 #define ImplicitOps (OpImplicit << DstShift)
82 #define DstReg (OpReg << DstShift)
83 #define DstMem (OpMem << DstShift)
84 #define DstAcc (OpAcc << DstShift)
85 #define DstDI (OpDI << DstShift)
86 #define DstMem64 (OpMem64 << DstShift)
87 #define DstImmUByte (OpImmUByte << DstShift)
88 #define DstDX (OpDX << DstShift)
89 #define DstMask (OpMask << DstShift)
90 /* Source operand type. */
92 #define SrcNone (OpNone << SrcShift)
93 #define SrcReg (OpReg << SrcShift)
94 #define SrcMem (OpMem << SrcShift)
95 #define SrcMem16 (OpMem16 << SrcShift)
96 #define SrcMem32 (OpMem32 << SrcShift)
97 #define SrcImm (OpImm << SrcShift)
98 #define SrcImmByte (OpImmByte << SrcShift)
99 #define SrcOne (OpOne << SrcShift)
100 #define SrcImmUByte (OpImmUByte << SrcShift)
101 #define SrcImmU (OpImmU << SrcShift)
102 #define SrcSI (OpSI << SrcShift)
103 #define SrcXLat (OpXLat << SrcShift)
104 #define SrcImmFAddr (OpImmFAddr << SrcShift)
105 #define SrcMemFAddr (OpMemFAddr << SrcShift)
106 #define SrcAcc (OpAcc << SrcShift)
107 #define SrcImmU16 (OpImmU16 << SrcShift)
108 #define SrcImm64 (OpImm64 << SrcShift)
109 #define SrcDX (OpDX << SrcShift)
110 #define SrcMem8 (OpMem8 << SrcShift)
111 #define SrcMask (OpMask << SrcShift)
112 #define BitOp (1<<11)
113 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
114 #define String (1<<13) /* String instruction (rep capable) */
115 #define Stack (1<<14) /* Stack instruction (push/pop) */
116 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
117 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
118 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
119 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
120 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
121 #define Escape (5<<15) /* Escape to coprocessor instruction */
122 #define Sse (1<<18) /* SSE Vector instruction */
123 /* Generic ModRM decode. */
124 #define ModRM (1<<19)
125 /* Destination is only written; never read. */
128 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
129 #define VendorSpecific (1<<22) /* Vendor specific instruction */
130 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132 #define Undefined (1<<25) /* No Such Instruction */
133 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
134 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
136 #define PageTable (1 << 29) /* instruction used to write page table */
137 #define NotImpl (1 << 30) /* instruction is not implemented */
138 /* Source 2 operand type */
139 #define Src2Shift (31)
140 #define Src2None (OpNone << Src2Shift)
141 #define Src2CL (OpCL << Src2Shift)
142 #define Src2ImmByte (OpImmByte << Src2Shift)
143 #define Src2One (OpOne << Src2Shift)
144 #define Src2Imm (OpImm << Src2Shift)
145 #define Src2ES (OpES << Src2Shift)
146 #define Src2CS (OpCS << Src2Shift)
147 #define Src2SS (OpSS << Src2Shift)
148 #define Src2DS (OpDS << Src2Shift)
149 #define Src2FS (OpFS << Src2Shift)
150 #define Src2GS (OpGS << Src2Shift)
151 #define Src2Mask (OpMask << Src2Shift)
152 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
153 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
154 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
155 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
156 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
157 #define NoWrite ((u64)1 << 45) /* No writeback */
159 #define X2(x...) x, x
160 #define X3(x...) X2(x), x
161 #define X4(x...) X2(x), X2(x)
162 #define X5(x...) X4(x), x
163 #define X6(x...) X4(x), X2(x)
164 #define X7(x...) X4(x), X3(x)
165 #define X8(x...) X4(x), X4(x)
166 #define X16(x...) X8(x), X8(x)
168 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
169 #define FASTOP_SIZE 8
172 * fastop functions have a special calling convention:
174 * dst: [rdx]:rax (in/out)
177 * flags: rflags (in/out)
179 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
180 * different operand sizes can be reached by calculation, rather than a jump
181 * table (which would be bigger than the code).
183 * fastop functions are declared as taking a never-defined fastop parameter,
184 * so they can't be called from C directly.
193 int (*execute)(struct x86_emulate_ctxt *ctxt);
194 const struct opcode *group;
195 const struct group_dual *gdual;
196 const struct gprefix *gprefix;
197 const struct escape *esc;
198 void (*fastop)(struct fastop *fake);
200 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
204 struct opcode mod012[8];
205 struct opcode mod3[8];
209 struct opcode pfx_no;
210 struct opcode pfx_66;
211 struct opcode pfx_f2;
212 struct opcode pfx_f3;
217 struct opcode high[64];
220 /* EFLAGS bit definitions. */
221 #define EFLG_ID (1<<21)
222 #define EFLG_VIP (1<<20)
223 #define EFLG_VIF (1<<19)
224 #define EFLG_AC (1<<18)
225 #define EFLG_VM (1<<17)
226 #define EFLG_RF (1<<16)
227 #define EFLG_IOPL (3<<12)
228 #define EFLG_NT (1<<14)
229 #define EFLG_OF (1<<11)
230 #define EFLG_DF (1<<10)
231 #define EFLG_IF (1<<9)
232 #define EFLG_TF (1<<8)
233 #define EFLG_SF (1<<7)
234 #define EFLG_ZF (1<<6)
235 #define EFLG_AF (1<<4)
236 #define EFLG_PF (1<<2)
237 #define EFLG_CF (1<<0)
239 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
240 #define EFLG_RESERVED_ONE_MASK 2
242 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
244 if (!(ctxt->regs_valid & (1 << nr))) {
245 ctxt->regs_valid |= 1 << nr;
246 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
248 return ctxt->_regs[nr];
251 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 ctxt->regs_valid |= 1 << nr;
254 ctxt->regs_dirty |= 1 << nr;
255 return &ctxt->_regs[nr];
258 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
261 return reg_write(ctxt, nr);
264 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
268 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
269 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
272 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
274 ctxt->regs_dirty = 0;
275 ctxt->regs_valid = 0;
279 * Instruction emulation:
280 * Most instructions are emulated directly via a fragment of inline assembly
281 * code. This allows us to save/restore EFLAGS and thus very easily pick up
282 * any modified flags.
285 #if defined(CONFIG_X86_64)
286 #define _LO32 "k" /* force 32-bit operand */
287 #define _STK "%%rsp" /* stack pointer */
288 #elif defined(__i386__)
289 #define _LO32 "" /* force 32-bit operand */
290 #define _STK "%%esp" /* stack pointer */
294 * These EFLAGS bits are restored from saved value during emulation, and
295 * any changes are written back to the saved value after emulation.
297 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 /* Before executing instruction: restore necessary bits in EFLAGS. */
300 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
301 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
302 "movl %"_sav",%"_LO32 _tmp"; " \
305 "movl %"_msk",%"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
308 "notl %"_LO32 _tmp"; " \
309 "andl %"_LO32 _tmp",("_STK"); " \
310 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
312 "orl %"_LO32 _tmp",("_STK"); " \
316 /* After executing instruction: write-back necessary bits in EFLAGS. */
317 #define _POST_EFLAGS(_sav, _msk, _tmp) \
318 /* _sav |= EFLAGS & _msk; */ \
321 "andl %"_msk",%"_LO32 _tmp"; " \
322 "orl %"_LO32 _tmp",%"_sav"; "
330 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
332 __asm__ __volatile__ ( \
333 _PRE_EFLAGS("0", "4", "2") \
334 _op _suffix " %"_x"3,%1; " \
335 _POST_EFLAGS("0", "4", "2") \
336 : "=m" ((ctxt)->eflags), \
337 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
339 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
343 /* Raw emulation: instruction has two explicit operands. */
344 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
346 unsigned long _tmp; \
348 switch ((ctxt)->dst.bytes) { \
350 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
353 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
356 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
361 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
363 unsigned long _tmp; \
364 switch ((ctxt)->dst.bytes) { \
366 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
369 __emulate_2op_nobyte(ctxt, _op, \
370 _wx, _wy, _lx, _ly, _qx, _qy); \
375 /* Source operand is byte-sized and may be restricted to just %cl. */
376 #define emulate_2op_SrcB(ctxt, _op) \
377 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
379 /* Source operand is byte, word, long or quad sized. */
380 #define emulate_2op_SrcV(ctxt, _op) \
381 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
383 /* Source operand is word, long or quad sized. */
384 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
385 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
387 /* Instruction has three operands and one operand is stored in ECX register */
388 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
390 unsigned long _tmp; \
391 _type _clv = (ctxt)->src2.val; \
392 _type _srcv = (ctxt)->src.val; \
393 _type _dstv = (ctxt)->dst.val; \
395 __asm__ __volatile__ ( \
396 _PRE_EFLAGS("0", "5", "2") \
397 _op _suffix " %4,%1 \n" \
398 _POST_EFLAGS("0", "5", "2") \
399 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
400 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
403 (ctxt)->src2.val = (unsigned long) _clv; \
404 (ctxt)->src2.val = (unsigned long) _srcv; \
405 (ctxt)->dst.val = (unsigned long) _dstv; \
408 #define emulate_2op_cl(ctxt, _op) \
410 switch ((ctxt)->dst.bytes) { \
412 __emulate_2op_cl(ctxt, _op, "w", u16); \
415 __emulate_2op_cl(ctxt, _op, "l", u32); \
418 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
423 #define __emulate_1op(ctxt, _op, _suffix) \
425 unsigned long _tmp; \
427 __asm__ __volatile__ ( \
428 _PRE_EFLAGS("0", "3", "2") \
429 _op _suffix " %1; " \
430 _POST_EFLAGS("0", "3", "2") \
431 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
433 : "i" (EFLAGS_MASK)); \
436 /* Instruction has only one explicit operand (no source operand). */
437 #define emulate_1op(ctxt, _op) \
439 switch ((ctxt)->dst.bytes) { \
440 case 1: __emulate_1op(ctxt, _op, "b"); break; \
441 case 2: __emulate_1op(ctxt, _op, "w"); break; \
442 case 4: __emulate_1op(ctxt, _op, "l"); break; \
443 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
447 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
449 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
450 #define FOP_RET "ret \n\t"
452 #define FOP_START(op) \
453 extern void em_##op(struct fastop *fake); \
454 asm(".pushsection .text, \"ax\" \n\t" \
455 ".global em_" #op " \n\t" \
462 #define FOPNOP() FOP_ALIGN FOP_RET
464 #define FOP1E(op, dst) \
465 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
467 #define FASTOP1(op) \
472 ON64(FOP1E(op##q, rax)) \
475 #define FOP2E(op, dst, src) \
476 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
478 #define FASTOP2(op) \
480 FOP2E(op##b, al, bl) \
481 FOP2E(op##w, ax, bx) \
482 FOP2E(op##l, eax, ebx) \
483 ON64(FOP2E(op##q, rax, rbx)) \
486 /* 2 operand, word only */
487 #define FASTOP2W(op) \
490 FOP2E(op##w, ax, bx) \
491 FOP2E(op##l, eax, ebx) \
492 ON64(FOP2E(op##q, rax, rbx)) \
495 /* 2 operand, src is CL */
496 #define FASTOP2CL(op) \
498 FOP2E(op##b, al, cl) \
499 FOP2E(op##w, ax, cl) \
500 FOP2E(op##l, eax, cl) \
501 ON64(FOP2E(op##q, rax, cl)) \
504 #define FOP3E(op, dst, src, src2) \
505 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
507 /* 3-operand, word-only, src2=cl */
508 #define FASTOP3WCL(op) \
511 FOP3E(op##w, ax, bx, cl) \
512 FOP3E(op##l, eax, ebx, cl) \
513 ON64(FOP3E(op##q, rax, rbx, cl)) \
516 /* Special case for SETcc - 1 instruction per cc */
517 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
538 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
541 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
543 unsigned long _tmp; \
544 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
545 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
547 __asm__ __volatile__ ( \
548 _PRE_EFLAGS("0", "5", "1") \
550 _op _suffix " %6; " \
552 _POST_EFLAGS("0", "5", "1") \
553 ".pushsection .fixup,\"ax\" \n\t" \
554 "3: movb $1, %4 \n\t" \
557 _ASM_EXTABLE(1b, 3b) \
558 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
559 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
560 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
563 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
564 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
566 switch((ctxt)->src.bytes) { \
568 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
571 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
574 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
577 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
582 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
583 enum x86_intercept intercept,
584 enum x86_intercept_stage stage)
586 struct x86_instruction_info info = {
587 .intercept = intercept,
588 .rep_prefix = ctxt->rep_prefix,
589 .modrm_mod = ctxt->modrm_mod,
590 .modrm_reg = ctxt->modrm_reg,
591 .modrm_rm = ctxt->modrm_rm,
592 .src_val = ctxt->src.val64,
593 .src_bytes = ctxt->src.bytes,
594 .dst_bytes = ctxt->dst.bytes,
595 .ad_bytes = ctxt->ad_bytes,
596 .next_rip = ctxt->eip,
599 return ctxt->ops->intercept(ctxt, &info, stage);
602 static void assign_masked(ulong *dest, ulong src, ulong mask)
604 *dest = (*dest & ~mask) | (src & mask);
607 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
609 return (1UL << (ctxt->ad_bytes << 3)) - 1;
612 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
615 struct desc_struct ss;
617 if (ctxt->mode == X86EMUL_MODE_PROT64)
619 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
620 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
623 static int stack_size(struct x86_emulate_ctxt *ctxt)
625 return (__fls(stack_mask(ctxt)) + 1) >> 3;
628 /* Access/update address held in a register, based on addressing mode. */
629 static inline unsigned long
630 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
632 if (ctxt->ad_bytes == sizeof(unsigned long))
635 return reg & ad_mask(ctxt);
638 static inline unsigned long
639 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
641 return address_mask(ctxt, reg);
644 static void masked_increment(ulong *reg, ulong mask, int inc)
646 assign_masked(reg, *reg + inc, mask);
650 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
654 if (ctxt->ad_bytes == sizeof(unsigned long))
657 mask = ad_mask(ctxt);
658 masked_increment(reg, mask, inc);
661 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
663 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
666 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
668 register_address_increment(ctxt, &ctxt->_eip, rel);
671 static u32 desc_limit_scaled(struct desc_struct *desc)
673 u32 limit = get_desc_limit(desc);
675 return desc->g ? (limit << 12) | 0xfff : limit;
678 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
680 ctxt->has_seg_override = true;
681 ctxt->seg_override = seg;
684 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
686 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
689 return ctxt->ops->get_cached_segment_base(ctxt, seg);
692 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
694 if (!ctxt->has_seg_override)
697 return ctxt->seg_override;
700 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
701 u32 error, bool valid)
703 ctxt->exception.vector = vec;
704 ctxt->exception.error_code = error;
705 ctxt->exception.error_code_valid = valid;
706 return X86EMUL_PROPAGATE_FAULT;
709 static int emulate_db(struct x86_emulate_ctxt *ctxt)
711 return emulate_exception(ctxt, DB_VECTOR, 0, false);
714 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
716 return emulate_exception(ctxt, GP_VECTOR, err, true);
719 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
721 return emulate_exception(ctxt, SS_VECTOR, err, true);
724 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
726 return emulate_exception(ctxt, UD_VECTOR, 0, false);
729 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
731 return emulate_exception(ctxt, TS_VECTOR, err, true);
734 static int emulate_de(struct x86_emulate_ctxt *ctxt)
736 return emulate_exception(ctxt, DE_VECTOR, 0, false);
739 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
741 return emulate_exception(ctxt, NM_VECTOR, 0, false);
744 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
747 struct desc_struct desc;
749 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
753 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
758 struct desc_struct desc;
760 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
761 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
765 * x86 defines three classes of vector instructions: explicitly
766 * aligned, explicitly unaligned, and the rest, which change behaviour
767 * depending on whether they're AVX encoded or not.
769 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
770 * subject to the same check.
772 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
774 if (likely(size < 16))
777 if (ctxt->d & Aligned)
779 else if (ctxt->d & Unaligned)
781 else if (ctxt->d & Avx)
787 static int __linearize(struct x86_emulate_ctxt *ctxt,
788 struct segmented_address addr,
789 unsigned size, bool write, bool fetch,
792 struct desc_struct desc;
799 la = seg_base(ctxt, addr.seg) + addr.ea;
800 switch (ctxt->mode) {
801 case X86EMUL_MODE_PROT64:
802 if (((signed long)la << 16) >> 16 != la)
803 return emulate_gp(ctxt, 0);
806 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
810 /* code segment in protected mode or read-only data segment */
811 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
812 || !(desc.type & 2)) && write)
814 /* unreadable code segment */
815 if (!fetch && (desc.type & 8) && !(desc.type & 2))
817 lim = desc_limit_scaled(&desc);
818 if ((desc.type & 8) || !(desc.type & 4)) {
819 /* expand-up segment */
820 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
823 /* expand-down segment */
824 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
826 lim = desc.d ? 0xffffffff : 0xffff;
827 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
830 cpl = ctxt->ops->cpl(ctxt);
831 if (!(desc.type & 8)) {
835 } else if ((desc.type & 8) && !(desc.type & 4)) {
836 /* nonconforming code segment */
839 } else if ((desc.type & 8) && (desc.type & 4)) {
840 /* conforming code segment */
846 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
848 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
849 return emulate_gp(ctxt, 0);
851 return X86EMUL_CONTINUE;
853 if (addr.seg == VCPU_SREG_SS)
854 return emulate_ss(ctxt, sel);
856 return emulate_gp(ctxt, sel);
859 static int linearize(struct x86_emulate_ctxt *ctxt,
860 struct segmented_address addr,
861 unsigned size, bool write,
864 return __linearize(ctxt, addr, size, write, false, linear);
868 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
869 struct segmented_address addr,
876 rc = linearize(ctxt, addr, size, false, &linear);
877 if (rc != X86EMUL_CONTINUE)
879 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
883 * Fetch the next byte of the instruction being emulated which is pointed to
884 * by ctxt->_eip, then increment ctxt->_eip.
886 * Also prefetch the remaining bytes of the instruction without crossing page
887 * boundary if they are not in fetch_cache yet.
889 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
891 struct fetch_cache *fc = &ctxt->fetch;
895 if (ctxt->_eip == fc->end) {
896 unsigned long linear;
897 struct segmented_address addr = { .seg = VCPU_SREG_CS,
899 cur_size = fc->end - fc->start;
900 size = min(15UL - cur_size,
901 PAGE_SIZE - offset_in_page(ctxt->_eip));
902 rc = __linearize(ctxt, addr, size, false, true, &linear);
903 if (unlikely(rc != X86EMUL_CONTINUE))
905 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
906 size, &ctxt->exception);
907 if (unlikely(rc != X86EMUL_CONTINUE))
911 *dest = fc->data[ctxt->_eip - fc->start];
913 return X86EMUL_CONTINUE;
916 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
917 void *dest, unsigned size)
921 /* x86 instructions are limited to 15 bytes. */
922 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
923 return X86EMUL_UNHANDLEABLE;
925 rc = do_insn_fetch_byte(ctxt, dest++);
926 if (rc != X86EMUL_CONTINUE)
929 return X86EMUL_CONTINUE;
932 /* Fetch next part of the instruction being emulated. */
933 #define insn_fetch(_type, _ctxt) \
934 ({ unsigned long _x; \
935 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
936 if (rc != X86EMUL_CONTINUE) \
941 #define insn_fetch_arr(_arr, _size, _ctxt) \
942 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
943 if (rc != X86EMUL_CONTINUE) \
948 * Given the 'reg' portion of a ModRM byte, and a register block, return a
949 * pointer into the block that addresses the relevant register.
950 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
952 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
957 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
958 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
960 p = reg_rmw(ctxt, modrm_reg);
964 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
965 struct segmented_address addr,
966 u16 *size, unsigned long *address, int op_bytes)
973 rc = segmented_read_std(ctxt, addr, size, 2);
974 if (rc != X86EMUL_CONTINUE)
977 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1016 static u8 test_cc(unsigned int condition, unsigned long flags)
1019 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1021 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1022 asm("push %[flags]; popf; call *%[fastop]"
1023 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1027 static void fetch_register_operand(struct operand *op)
1029 switch (op->bytes) {
1031 op->val = *(u8 *)op->addr.reg;
1034 op->val = *(u16 *)op->addr.reg;
1037 op->val = *(u32 *)op->addr.reg;
1040 op->val = *(u64 *)op->addr.reg;
1045 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1047 ctxt->ops->get_fpu(ctxt);
1049 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1050 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1051 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1052 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1053 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1054 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1055 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1056 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1057 #ifdef CONFIG_X86_64
1058 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1059 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1060 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1061 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1062 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1063 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1064 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1065 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1069 ctxt->ops->put_fpu(ctxt);
1072 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1075 ctxt->ops->get_fpu(ctxt);
1077 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1078 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1079 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1080 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1081 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1082 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1083 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1084 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1085 #ifdef CONFIG_X86_64
1086 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1087 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1088 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1089 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1090 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1091 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1092 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1093 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1097 ctxt->ops->put_fpu(ctxt);
1100 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1102 ctxt->ops->get_fpu(ctxt);
1104 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1105 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1106 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1107 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1108 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1109 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1110 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1111 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1114 ctxt->ops->put_fpu(ctxt);
1117 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1119 ctxt->ops->get_fpu(ctxt);
1121 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1122 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1123 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1124 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1125 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1126 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1127 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1128 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1131 ctxt->ops->put_fpu(ctxt);
1134 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1136 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1137 return emulate_nm(ctxt);
1139 ctxt->ops->get_fpu(ctxt);
1140 asm volatile("fninit");
1141 ctxt->ops->put_fpu(ctxt);
1142 return X86EMUL_CONTINUE;
1145 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1149 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1150 return emulate_nm(ctxt);
1152 ctxt->ops->get_fpu(ctxt);
1153 asm volatile("fnstcw %0": "+m"(fcw));
1154 ctxt->ops->put_fpu(ctxt);
1156 /* force 2 byte destination */
1157 ctxt->dst.bytes = 2;
1158 ctxt->dst.val = fcw;
1160 return X86EMUL_CONTINUE;
1163 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1167 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1168 return emulate_nm(ctxt);
1170 ctxt->ops->get_fpu(ctxt);
1171 asm volatile("fnstsw %0": "+m"(fsw));
1172 ctxt->ops->put_fpu(ctxt);
1174 /* force 2 byte destination */
1175 ctxt->dst.bytes = 2;
1176 ctxt->dst.val = fsw;
1178 return X86EMUL_CONTINUE;
1181 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1184 unsigned reg = ctxt->modrm_reg;
1185 int highbyte_regs = ctxt->rex_prefix == 0;
1187 if (!(ctxt->d & ModRM))
1188 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1190 if (ctxt->d & Sse) {
1194 read_sse_reg(ctxt, &op->vec_val, reg);
1197 if (ctxt->d & Mmx) {
1206 if (ctxt->d & ByteOp) {
1207 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1210 op->addr.reg = decode_register(ctxt, reg, 0);
1211 op->bytes = ctxt->op_bytes;
1213 fetch_register_operand(op);
1214 op->orig_val = op->val;
1217 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1219 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1220 ctxt->modrm_seg = VCPU_SREG_SS;
1223 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1227 int index_reg = 0, base_reg = 0, scale;
1228 int rc = X86EMUL_CONTINUE;
1231 if (ctxt->rex_prefix) {
1232 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1233 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1234 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1237 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1238 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1239 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1240 ctxt->modrm_seg = VCPU_SREG_DS;
1242 if (ctxt->modrm_mod == 3) {
1244 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1245 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1246 if (ctxt->d & Sse) {
1249 op->addr.xmm = ctxt->modrm_rm;
1250 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1253 if (ctxt->d & Mmx) {
1256 op->addr.xmm = ctxt->modrm_rm & 7;
1259 fetch_register_operand(op);
1265 if (ctxt->ad_bytes == 2) {
1266 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1267 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1268 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1269 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1271 /* 16-bit ModR/M decode. */
1272 switch (ctxt->modrm_mod) {
1274 if (ctxt->modrm_rm == 6)
1275 modrm_ea += insn_fetch(u16, ctxt);
1278 modrm_ea += insn_fetch(s8, ctxt);
1281 modrm_ea += insn_fetch(u16, ctxt);
1284 switch (ctxt->modrm_rm) {
1286 modrm_ea += bx + si;
1289 modrm_ea += bx + di;
1292 modrm_ea += bp + si;
1295 modrm_ea += bp + di;
1304 if (ctxt->modrm_mod != 0)
1311 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1312 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1313 ctxt->modrm_seg = VCPU_SREG_SS;
1314 modrm_ea = (u16)modrm_ea;
1316 /* 32/64-bit ModR/M decode. */
1317 if ((ctxt->modrm_rm & 7) == 4) {
1318 sib = insn_fetch(u8, ctxt);
1319 index_reg |= (sib >> 3) & 7;
1320 base_reg |= sib & 7;
1323 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1324 modrm_ea += insn_fetch(s32, ctxt);
1326 modrm_ea += reg_read(ctxt, base_reg);
1327 adjust_modrm_seg(ctxt, base_reg);
1330 modrm_ea += reg_read(ctxt, index_reg) << scale;
1331 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1332 if (ctxt->mode == X86EMUL_MODE_PROT64)
1333 ctxt->rip_relative = 1;
1335 base_reg = ctxt->modrm_rm;
1336 modrm_ea += reg_read(ctxt, base_reg);
1337 adjust_modrm_seg(ctxt, base_reg);
1339 switch (ctxt->modrm_mod) {
1341 if (ctxt->modrm_rm == 5)
1342 modrm_ea += insn_fetch(s32, ctxt);
1345 modrm_ea += insn_fetch(s8, ctxt);
1348 modrm_ea += insn_fetch(s32, ctxt);
1352 op->addr.mem.ea = modrm_ea;
1357 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1360 int rc = X86EMUL_CONTINUE;
1363 switch (ctxt->ad_bytes) {
1365 op->addr.mem.ea = insn_fetch(u16, ctxt);
1368 op->addr.mem.ea = insn_fetch(u32, ctxt);
1371 op->addr.mem.ea = insn_fetch(u64, ctxt);
1378 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1382 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1383 mask = ~(ctxt->dst.bytes * 8 - 1);
1385 if (ctxt->src.bytes == 2)
1386 sv = (s16)ctxt->src.val & (s16)mask;
1387 else if (ctxt->src.bytes == 4)
1388 sv = (s32)ctxt->src.val & (s32)mask;
1390 ctxt->dst.addr.mem.ea += (sv >> 3);
1393 /* only subword offset */
1394 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1397 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1398 unsigned long addr, void *dest, unsigned size)
1401 struct read_cache *mc = &ctxt->mem_read;
1403 if (mc->pos < mc->end)
1406 WARN_ON((mc->end + size) >= sizeof(mc->data));
1408 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1410 if (rc != X86EMUL_CONTINUE)
1416 memcpy(dest, mc->data + mc->pos, size);
1418 return X86EMUL_CONTINUE;
1421 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1422 struct segmented_address addr,
1429 rc = linearize(ctxt, addr, size, false, &linear);
1430 if (rc != X86EMUL_CONTINUE)
1432 return read_emulated(ctxt, linear, data, size);
1435 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1436 struct segmented_address addr,
1443 rc = linearize(ctxt, addr, size, true, &linear);
1444 if (rc != X86EMUL_CONTINUE)
1446 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1450 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1451 struct segmented_address addr,
1452 const void *orig_data, const void *data,
1458 rc = linearize(ctxt, addr, size, true, &linear);
1459 if (rc != X86EMUL_CONTINUE)
1461 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1462 size, &ctxt->exception);
1465 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1466 unsigned int size, unsigned short port,
1469 struct read_cache *rc = &ctxt->io_read;
1471 if (rc->pos == rc->end) { /* refill pio read ahead */
1472 unsigned int in_page, n;
1473 unsigned int count = ctxt->rep_prefix ?
1474 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1475 in_page = (ctxt->eflags & EFLG_DF) ?
1476 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1477 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1478 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1482 rc->pos = rc->end = 0;
1483 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1488 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1489 ctxt->dst.data = rc->data + rc->pos;
1490 ctxt->dst.type = OP_MEM_STR;
1491 ctxt->dst.count = (rc->end - rc->pos) / size;
1494 memcpy(dest, rc->data + rc->pos, size);
1500 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1501 u16 index, struct desc_struct *desc)
1506 ctxt->ops->get_idt(ctxt, &dt);
1508 if (dt.size < index * 8 + 7)
1509 return emulate_gp(ctxt, index << 3 | 0x2);
1511 addr = dt.address + index * 8;
1512 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1516 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1517 u16 selector, struct desc_ptr *dt)
1519 const struct x86_emulate_ops *ops = ctxt->ops;
1521 if (selector & 1 << 2) {
1522 struct desc_struct desc;
1525 memset (dt, 0, sizeof *dt);
1526 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1529 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1530 dt->address = get_desc_base(&desc);
1532 ops->get_gdt(ctxt, dt);
1535 /* allowed just for 8 bytes segments */
1536 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1537 u16 selector, struct desc_struct *desc,
1541 u16 index = selector >> 3;
1544 get_descriptor_table_ptr(ctxt, selector, &dt);
1546 if (dt.size < index * 8 + 7)
1547 return emulate_gp(ctxt, selector & 0xfffc);
1549 *desc_addr_p = addr = dt.address + index * 8;
1550 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1554 /* allowed just for 8 bytes segments */
1555 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1556 u16 selector, struct desc_struct *desc)
1559 u16 index = selector >> 3;
1562 get_descriptor_table_ptr(ctxt, selector, &dt);
1564 if (dt.size < index * 8 + 7)
1565 return emulate_gp(ctxt, selector & 0xfffc);
1567 addr = dt.address + index * 8;
1568 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1572 /* Does not support long mode */
1573 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1574 u16 selector, int seg)
1576 struct desc_struct seg_desc, old_desc;
1578 unsigned err_vec = GP_VECTOR;
1580 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1585 memset(&seg_desc, 0, sizeof seg_desc);
1587 if (ctxt->mode == X86EMUL_MODE_REAL) {
1588 /* set real mode segment descriptor (keep limit etc. for
1590 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1591 set_desc_base(&seg_desc, selector << 4);
1593 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1594 /* VM86 needs a clean new segment descriptor */
1595 set_desc_base(&seg_desc, selector << 4);
1596 set_desc_limit(&seg_desc, 0xffff);
1605 cpl = ctxt->ops->cpl(ctxt);
1607 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1608 if ((seg == VCPU_SREG_CS
1609 || (seg == VCPU_SREG_SS
1610 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1611 || seg == VCPU_SREG_TR)
1615 /* TR should be in GDT only */
1616 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1619 if (null_selector) /* for NULL selector skip all following checks */
1622 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1623 if (ret != X86EMUL_CONTINUE)
1626 err_code = selector & 0xfffc;
1627 err_vec = GP_VECTOR;
1629 /* can't load system descriptor into segment selector */
1630 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1634 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1643 * segment is not a writable data segment or segment
1644 * selector's RPL != CPL or segment selector's RPL != CPL
1646 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1650 if (!(seg_desc.type & 8))
1653 if (seg_desc.type & 4) {
1659 if (rpl > cpl || dpl != cpl)
1662 /* CS(RPL) <- CPL */
1663 selector = (selector & 0xfffc) | cpl;
1666 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1668 old_desc = seg_desc;
1669 seg_desc.type |= 2; /* busy */
1670 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1671 sizeof(seg_desc), &ctxt->exception);
1672 if (ret != X86EMUL_CONTINUE)
1675 case VCPU_SREG_LDTR:
1676 if (seg_desc.s || seg_desc.type != 2)
1679 default: /* DS, ES, FS, or GS */
1681 * segment is not a data or readable code segment or
1682 * ((segment is a data or nonconforming code segment)
1683 * and (both RPL and CPL > DPL))
1685 if ((seg_desc.type & 0xa) == 0x8 ||
1686 (((seg_desc.type & 0xc) != 0xc) &&
1687 (rpl > dpl && cpl > dpl)))
1693 /* mark segment as accessed */
1695 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1696 if (ret != X86EMUL_CONTINUE)
1700 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1701 return X86EMUL_CONTINUE;
1703 emulate_exception(ctxt, err_vec, err_code, true);
1704 return X86EMUL_PROPAGATE_FAULT;
1707 static void write_register_operand(struct operand *op)
1709 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1710 switch (op->bytes) {
1712 *(u8 *)op->addr.reg = (u8)op->val;
1715 *(u16 *)op->addr.reg = (u16)op->val;
1718 *op->addr.reg = (u32)op->val;
1719 break; /* 64b: zero-extend */
1721 *op->addr.reg = op->val;
1726 static int writeback(struct x86_emulate_ctxt *ctxt)
1730 if (ctxt->d & NoWrite)
1731 return X86EMUL_CONTINUE;
1733 switch (ctxt->dst.type) {
1735 write_register_operand(&ctxt->dst);
1738 if (ctxt->lock_prefix)
1739 rc = segmented_cmpxchg(ctxt,
1741 &ctxt->dst.orig_val,
1745 rc = segmented_write(ctxt,
1749 if (rc != X86EMUL_CONTINUE)
1753 rc = segmented_write(ctxt,
1756 ctxt->dst.bytes * ctxt->dst.count);
1757 if (rc != X86EMUL_CONTINUE)
1761 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1764 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1772 return X86EMUL_CONTINUE;
1775 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1777 struct segmented_address addr;
1779 rsp_increment(ctxt, -bytes);
1780 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1781 addr.seg = VCPU_SREG_SS;
1783 return segmented_write(ctxt, addr, data, bytes);
1786 static int em_push(struct x86_emulate_ctxt *ctxt)
1788 /* Disable writeback. */
1789 ctxt->dst.type = OP_NONE;
1790 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1793 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1794 void *dest, int len)
1797 struct segmented_address addr;
1799 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1800 addr.seg = VCPU_SREG_SS;
1801 rc = segmented_read(ctxt, addr, dest, len);
1802 if (rc != X86EMUL_CONTINUE)
1805 rsp_increment(ctxt, len);
1809 static int em_pop(struct x86_emulate_ctxt *ctxt)
1811 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1814 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1815 void *dest, int len)
1818 unsigned long val, change_mask;
1819 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1820 int cpl = ctxt->ops->cpl(ctxt);
1822 rc = emulate_pop(ctxt, &val, len);
1823 if (rc != X86EMUL_CONTINUE)
1826 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1827 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1829 switch(ctxt->mode) {
1830 case X86EMUL_MODE_PROT64:
1831 case X86EMUL_MODE_PROT32:
1832 case X86EMUL_MODE_PROT16:
1834 change_mask |= EFLG_IOPL;
1836 change_mask |= EFLG_IF;
1838 case X86EMUL_MODE_VM86:
1840 return emulate_gp(ctxt, 0);
1841 change_mask |= EFLG_IF;
1843 default: /* real mode */
1844 change_mask |= (EFLG_IOPL | EFLG_IF);
1848 *(unsigned long *)dest =
1849 (ctxt->eflags & ~change_mask) | (val & change_mask);
1854 static int em_popf(struct x86_emulate_ctxt *ctxt)
1856 ctxt->dst.type = OP_REG;
1857 ctxt->dst.addr.reg = &ctxt->eflags;
1858 ctxt->dst.bytes = ctxt->op_bytes;
1859 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1862 static int em_enter(struct x86_emulate_ctxt *ctxt)
1865 unsigned frame_size = ctxt->src.val;
1866 unsigned nesting_level = ctxt->src2.val & 31;
1870 return X86EMUL_UNHANDLEABLE;
1872 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1873 rc = push(ctxt, &rbp, stack_size(ctxt));
1874 if (rc != X86EMUL_CONTINUE)
1876 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1878 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1879 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1881 return X86EMUL_CONTINUE;
1884 static int em_leave(struct x86_emulate_ctxt *ctxt)
1886 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1888 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1891 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1893 int seg = ctxt->src2.val;
1895 ctxt->src.val = get_segment_selector(ctxt, seg);
1897 return em_push(ctxt);
1900 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1902 int seg = ctxt->src2.val;
1903 unsigned long selector;
1906 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1907 if (rc != X86EMUL_CONTINUE)
1910 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1914 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1916 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1917 int rc = X86EMUL_CONTINUE;
1918 int reg = VCPU_REGS_RAX;
1920 while (reg <= VCPU_REGS_RDI) {
1921 (reg == VCPU_REGS_RSP) ?
1922 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1925 if (rc != X86EMUL_CONTINUE)
1934 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1936 ctxt->src.val = (unsigned long)ctxt->eflags;
1937 return em_push(ctxt);
1940 static int em_popa(struct x86_emulate_ctxt *ctxt)
1942 int rc = X86EMUL_CONTINUE;
1943 int reg = VCPU_REGS_RDI;
1945 while (reg >= VCPU_REGS_RAX) {
1946 if (reg == VCPU_REGS_RSP) {
1947 rsp_increment(ctxt, ctxt->op_bytes);
1951 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1952 if (rc != X86EMUL_CONTINUE)
1959 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1961 const struct x86_emulate_ops *ops = ctxt->ops;
1968 /* TODO: Add limit checks */
1969 ctxt->src.val = ctxt->eflags;
1971 if (rc != X86EMUL_CONTINUE)
1974 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1976 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1978 if (rc != X86EMUL_CONTINUE)
1981 ctxt->src.val = ctxt->_eip;
1983 if (rc != X86EMUL_CONTINUE)
1986 ops->get_idt(ctxt, &dt);
1988 eip_addr = dt.address + (irq << 2);
1989 cs_addr = dt.address + (irq << 2) + 2;
1991 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1992 if (rc != X86EMUL_CONTINUE)
1995 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1996 if (rc != X86EMUL_CONTINUE)
1999 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2000 if (rc != X86EMUL_CONTINUE)
2008 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2012 invalidate_registers(ctxt);
2013 rc = __emulate_int_real(ctxt, irq);
2014 if (rc == X86EMUL_CONTINUE)
2015 writeback_registers(ctxt);
2019 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2021 switch(ctxt->mode) {
2022 case X86EMUL_MODE_REAL:
2023 return __emulate_int_real(ctxt, irq);
2024 case X86EMUL_MODE_VM86:
2025 case X86EMUL_MODE_PROT16:
2026 case X86EMUL_MODE_PROT32:
2027 case X86EMUL_MODE_PROT64:
2029 /* Protected mode interrupts unimplemented yet */
2030 return X86EMUL_UNHANDLEABLE;
2034 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2036 int rc = X86EMUL_CONTINUE;
2037 unsigned long temp_eip = 0;
2038 unsigned long temp_eflags = 0;
2039 unsigned long cs = 0;
2040 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2041 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2042 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2043 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2045 /* TODO: Add stack limit check */
2047 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2049 if (rc != X86EMUL_CONTINUE)
2052 if (temp_eip & ~0xffff)
2053 return emulate_gp(ctxt, 0);
2055 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2057 if (rc != X86EMUL_CONTINUE)
2060 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2062 if (rc != X86EMUL_CONTINUE)
2065 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2067 if (rc != X86EMUL_CONTINUE)
2070 ctxt->_eip = temp_eip;
2073 if (ctxt->op_bytes == 4)
2074 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2075 else if (ctxt->op_bytes == 2) {
2076 ctxt->eflags &= ~0xffff;
2077 ctxt->eflags |= temp_eflags;
2080 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2081 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2086 static int em_iret(struct x86_emulate_ctxt *ctxt)
2088 switch(ctxt->mode) {
2089 case X86EMUL_MODE_REAL:
2090 return emulate_iret_real(ctxt);
2091 case X86EMUL_MODE_VM86:
2092 case X86EMUL_MODE_PROT16:
2093 case X86EMUL_MODE_PROT32:
2094 case X86EMUL_MODE_PROT64:
2096 /* iret from protected mode unimplemented yet */
2097 return X86EMUL_UNHANDLEABLE;
2101 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2106 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2108 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2109 if (rc != X86EMUL_CONTINUE)
2113 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2114 return X86EMUL_CONTINUE;
2117 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2121 emulate_1op_rax_rdx(ctxt, "mul", ex);
2122 return X86EMUL_CONTINUE;
2125 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2129 emulate_1op_rax_rdx(ctxt, "imul", ex);
2130 return X86EMUL_CONTINUE;
2133 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2137 emulate_1op_rax_rdx(ctxt, "div", de);
2139 return emulate_de(ctxt);
2140 return X86EMUL_CONTINUE;
2143 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2147 emulate_1op_rax_rdx(ctxt, "idiv", de);
2149 return emulate_de(ctxt);
2150 return X86EMUL_CONTINUE;
2153 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2155 int rc = X86EMUL_CONTINUE;
2157 switch (ctxt->modrm_reg) {
2158 case 2: /* call near abs */ {
2160 old_eip = ctxt->_eip;
2161 ctxt->_eip = ctxt->src.val;
2162 ctxt->src.val = old_eip;
2166 case 4: /* jmp abs */
2167 ctxt->_eip = ctxt->src.val;
2169 case 5: /* jmp far */
2170 rc = em_jmp_far(ctxt);
2179 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2181 u64 old = ctxt->dst.orig_val64;
2183 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2184 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2185 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2186 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2187 ctxt->eflags &= ~EFLG_ZF;
2189 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2190 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2192 ctxt->eflags |= EFLG_ZF;
2194 return X86EMUL_CONTINUE;
2197 static int em_ret(struct x86_emulate_ctxt *ctxt)
2199 ctxt->dst.type = OP_REG;
2200 ctxt->dst.addr.reg = &ctxt->_eip;
2201 ctxt->dst.bytes = ctxt->op_bytes;
2202 return em_pop(ctxt);
2205 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2210 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2211 if (rc != X86EMUL_CONTINUE)
2213 if (ctxt->op_bytes == 4)
2214 ctxt->_eip = (u32)ctxt->_eip;
2215 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2216 if (rc != X86EMUL_CONTINUE)
2218 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2222 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2224 /* Save real source value, then compare EAX against destination. */
2225 ctxt->src.orig_val = ctxt->src.val;
2226 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2227 fastop(ctxt, em_cmp);
2229 if (ctxt->eflags & EFLG_ZF) {
2230 /* Success: write back to memory. */
2231 ctxt->dst.val = ctxt->src.orig_val;
2233 /* Failure: write the value we saw to EAX. */
2234 ctxt->dst.type = OP_REG;
2235 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2237 return X86EMUL_CONTINUE;
2240 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2242 int seg = ctxt->src2.val;
2246 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2248 rc = load_segment_descriptor(ctxt, sel, seg);
2249 if (rc != X86EMUL_CONTINUE)
2252 ctxt->dst.val = ctxt->src.val;
2257 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2258 struct desc_struct *cs, struct desc_struct *ss)
2260 cs->l = 0; /* will be adjusted later */
2261 set_desc_base(cs, 0); /* flat segment */
2262 cs->g = 1; /* 4kb granularity */
2263 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2264 cs->type = 0x0b; /* Read, Execute, Accessed */
2266 cs->dpl = 0; /* will be adjusted later */
2271 set_desc_base(ss, 0); /* flat segment */
2272 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2273 ss->g = 1; /* 4kb granularity */
2275 ss->type = 0x03; /* Read/Write, Accessed */
2276 ss->d = 1; /* 32bit stack segment */
2283 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2285 u32 eax, ebx, ecx, edx;
2288 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2289 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2290 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2291 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2294 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2296 const struct x86_emulate_ops *ops = ctxt->ops;
2297 u32 eax, ebx, ecx, edx;
2300 * syscall should always be enabled in longmode - so only become
2301 * vendor specific (cpuid) if other modes are active...
2303 if (ctxt->mode == X86EMUL_MODE_PROT64)
2308 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2310 * Intel ("GenuineIntel")
2311 * remark: Intel CPUs only support "syscall" in 64bit
2312 * longmode. Also an 64bit guest with a
2313 * 32bit compat-app running will #UD !! While this
2314 * behaviour can be fixed (by emulating) into AMD
2315 * response - CPUs of AMD can't behave like Intel.
2317 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2318 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2319 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2322 /* AMD ("AuthenticAMD") */
2323 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2324 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2325 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2328 /* AMD ("AMDisbetter!") */
2329 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2330 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2331 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2334 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2338 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2340 const struct x86_emulate_ops *ops = ctxt->ops;
2341 struct desc_struct cs, ss;
2346 /* syscall is not available in real mode */
2347 if (ctxt->mode == X86EMUL_MODE_REAL ||
2348 ctxt->mode == X86EMUL_MODE_VM86)
2349 return emulate_ud(ctxt);
2351 if (!(em_syscall_is_enabled(ctxt)))
2352 return emulate_ud(ctxt);
2354 ops->get_msr(ctxt, MSR_EFER, &efer);
2355 setup_syscalls_segments(ctxt, &cs, &ss);
2357 if (!(efer & EFER_SCE))
2358 return emulate_ud(ctxt);
2360 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2362 cs_sel = (u16)(msr_data & 0xfffc);
2363 ss_sel = (u16)(msr_data + 8);
2365 if (efer & EFER_LMA) {
2369 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2370 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2372 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2373 if (efer & EFER_LMA) {
2374 #ifdef CONFIG_X86_64
2375 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2378 ctxt->mode == X86EMUL_MODE_PROT64 ?
2379 MSR_LSTAR : MSR_CSTAR, &msr_data);
2380 ctxt->_eip = msr_data;
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~(msr_data | EFLG_RF);
2387 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2388 ctxt->_eip = (u32)msr_data;
2390 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2393 return X86EMUL_CONTINUE;
2396 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2398 const struct x86_emulate_ops *ops = ctxt->ops;
2399 struct desc_struct cs, ss;
2404 ops->get_msr(ctxt, MSR_EFER, &efer);
2405 /* inject #GP if in real mode */
2406 if (ctxt->mode == X86EMUL_MODE_REAL)
2407 return emulate_gp(ctxt, 0);
2410 * Not recognized on AMD in compat mode (but is recognized in legacy
2413 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2414 && !vendor_intel(ctxt))
2415 return emulate_ud(ctxt);
2417 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2418 * Therefore, we inject an #UD.
2420 if (ctxt->mode == X86EMUL_MODE_PROT64)
2421 return emulate_ud(ctxt);
2423 setup_syscalls_segments(ctxt, &cs, &ss);
2425 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2426 switch (ctxt->mode) {
2427 case X86EMUL_MODE_PROT32:
2428 if ((msr_data & 0xfffc) == 0x0)
2429 return emulate_gp(ctxt, 0);
2431 case X86EMUL_MODE_PROT64:
2432 if (msr_data == 0x0)
2433 return emulate_gp(ctxt, 0);
2439 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2440 cs_sel = (u16)msr_data;
2441 cs_sel &= ~SELECTOR_RPL_MASK;
2442 ss_sel = cs_sel + 8;
2443 ss_sel &= ~SELECTOR_RPL_MASK;
2444 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2449 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2450 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2452 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2453 ctxt->_eip = msr_data;
2455 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2456 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2458 return X86EMUL_CONTINUE;
2461 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2463 const struct x86_emulate_ops *ops = ctxt->ops;
2464 struct desc_struct cs, ss;
2467 u16 cs_sel = 0, ss_sel = 0;
2469 /* inject #GP if in real mode or Virtual 8086 mode */
2470 if (ctxt->mode == X86EMUL_MODE_REAL ||
2471 ctxt->mode == X86EMUL_MODE_VM86)
2472 return emulate_gp(ctxt, 0);
2474 setup_syscalls_segments(ctxt, &cs, &ss);
2476 if ((ctxt->rex_prefix & 0x8) != 0x0)
2477 usermode = X86EMUL_MODE_PROT64;
2479 usermode = X86EMUL_MODE_PROT32;
2483 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2485 case X86EMUL_MODE_PROT32:
2486 cs_sel = (u16)(msr_data + 16);
2487 if ((msr_data & 0xfffc) == 0x0)
2488 return emulate_gp(ctxt, 0);
2489 ss_sel = (u16)(msr_data + 24);
2491 case X86EMUL_MODE_PROT64:
2492 cs_sel = (u16)(msr_data + 32);
2493 if (msr_data == 0x0)
2494 return emulate_gp(ctxt, 0);
2495 ss_sel = cs_sel + 8;
2500 cs_sel |= SELECTOR_RPL_MASK;
2501 ss_sel |= SELECTOR_RPL_MASK;
2503 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2504 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2506 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2507 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2509 return X86EMUL_CONTINUE;
2512 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2515 if (ctxt->mode == X86EMUL_MODE_REAL)
2517 if (ctxt->mode == X86EMUL_MODE_VM86)
2519 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2520 return ctxt->ops->cpl(ctxt) > iopl;
2523 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2526 const struct x86_emulate_ops *ops = ctxt->ops;
2527 struct desc_struct tr_seg;
2530 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2531 unsigned mask = (1 << len) - 1;
2534 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2537 if (desc_limit_scaled(&tr_seg) < 103)
2539 base = get_desc_base(&tr_seg);
2540 #ifdef CONFIG_X86_64
2541 base |= ((u64)base3) << 32;
2543 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2544 if (r != X86EMUL_CONTINUE)
2546 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2548 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2549 if (r != X86EMUL_CONTINUE)
2551 if ((perm >> bit_idx) & mask)
2556 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2562 if (emulator_bad_iopl(ctxt))
2563 if (!emulator_io_port_access_allowed(ctxt, port, len))
2566 ctxt->perm_ok = true;
2571 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2572 struct tss_segment_16 *tss)
2574 tss->ip = ctxt->_eip;
2575 tss->flag = ctxt->eflags;
2576 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2577 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2578 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2579 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2580 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2581 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2582 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2583 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2585 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2586 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2587 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2588 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2589 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2592 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2593 struct tss_segment_16 *tss)
2597 ctxt->_eip = tss->ip;
2598 ctxt->eflags = tss->flag | 2;
2599 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2600 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2601 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2602 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2603 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2604 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2605 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2606 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2609 * SDM says that segment selectors are loaded before segment
2612 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2613 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2614 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2615 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2616 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2619 * Now load segment descriptors. If fault happens at this stage
2620 * it is handled in a context of new task
2622 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2626 if (ret != X86EMUL_CONTINUE)
2628 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2635 if (ret != X86EMUL_CONTINUE)
2638 return X86EMUL_CONTINUE;
2641 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2642 u16 tss_selector, u16 old_tss_sel,
2643 ulong old_tss_base, struct desc_struct *new_desc)
2645 const struct x86_emulate_ops *ops = ctxt->ops;
2646 struct tss_segment_16 tss_seg;
2648 u32 new_tss_base = get_desc_base(new_desc);
2650 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2652 if (ret != X86EMUL_CONTINUE)
2653 /* FIXME: need to provide precise fault address */
2656 save_state_to_tss16(ctxt, &tss_seg);
2658 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2660 if (ret != X86EMUL_CONTINUE)
2661 /* FIXME: need to provide precise fault address */
2664 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2666 if (ret != X86EMUL_CONTINUE)
2667 /* FIXME: need to provide precise fault address */
2670 if (old_tss_sel != 0xffff) {
2671 tss_seg.prev_task_link = old_tss_sel;
2673 ret = ops->write_std(ctxt, new_tss_base,
2674 &tss_seg.prev_task_link,
2675 sizeof tss_seg.prev_task_link,
2677 if (ret != X86EMUL_CONTINUE)
2678 /* FIXME: need to provide precise fault address */
2682 return load_state_from_tss16(ctxt, &tss_seg);
2685 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2686 struct tss_segment_32 *tss)
2688 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2689 tss->eip = ctxt->_eip;
2690 tss->eflags = ctxt->eflags;
2691 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2692 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2693 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2694 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2695 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2696 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2697 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2698 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2700 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2701 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2702 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2703 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2704 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2705 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2706 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2709 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2710 struct tss_segment_32 *tss)
2714 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2715 return emulate_gp(ctxt, 0);
2716 ctxt->_eip = tss->eip;
2717 ctxt->eflags = tss->eflags | 2;
2719 /* General purpose registers */
2720 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2721 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2722 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2723 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2724 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2725 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2726 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2727 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2730 * SDM says that segment selectors are loaded before segment
2733 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2734 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2735 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2736 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2737 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2738 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2739 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2742 * If we're switching between Protected Mode and VM86, we need to make
2743 * sure to update the mode before loading the segment descriptors so
2744 * that the selectors are interpreted correctly.
2746 * Need to get rflags to the vcpu struct immediately because it
2747 * influences the CPL which is checked at least when loading the segment
2748 * descriptors and when pushing an error code to the new kernel stack.
2750 * TODO Introduce a separate ctxt->ops->set_cpl callback
2752 if (ctxt->eflags & X86_EFLAGS_VM)
2753 ctxt->mode = X86EMUL_MODE_VM86;
2755 ctxt->mode = X86EMUL_MODE_PROT32;
2757 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2760 * Now load segment descriptors. If fault happenes at this stage
2761 * it is handled in a context of new task
2763 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2764 if (ret != X86EMUL_CONTINUE)
2766 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2767 if (ret != X86EMUL_CONTINUE)
2769 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2770 if (ret != X86EMUL_CONTINUE)
2772 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2776 if (ret != X86EMUL_CONTINUE)
2778 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2779 if (ret != X86EMUL_CONTINUE)
2781 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2782 if (ret != X86EMUL_CONTINUE)
2785 return X86EMUL_CONTINUE;
2788 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2789 u16 tss_selector, u16 old_tss_sel,
2790 ulong old_tss_base, struct desc_struct *new_desc)
2792 const struct x86_emulate_ops *ops = ctxt->ops;
2793 struct tss_segment_32 tss_seg;
2795 u32 new_tss_base = get_desc_base(new_desc);
2797 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2799 if (ret != X86EMUL_CONTINUE)
2800 /* FIXME: need to provide precise fault address */
2803 save_state_to_tss32(ctxt, &tss_seg);
2805 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2807 if (ret != X86EMUL_CONTINUE)
2808 /* FIXME: need to provide precise fault address */
2811 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2813 if (ret != X86EMUL_CONTINUE)
2814 /* FIXME: need to provide precise fault address */
2817 if (old_tss_sel != 0xffff) {
2818 tss_seg.prev_task_link = old_tss_sel;
2820 ret = ops->write_std(ctxt, new_tss_base,
2821 &tss_seg.prev_task_link,
2822 sizeof tss_seg.prev_task_link,
2824 if (ret != X86EMUL_CONTINUE)
2825 /* FIXME: need to provide precise fault address */
2829 return load_state_from_tss32(ctxt, &tss_seg);
2832 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2833 u16 tss_selector, int idt_index, int reason,
2834 bool has_error_code, u32 error_code)
2836 const struct x86_emulate_ops *ops = ctxt->ops;
2837 struct desc_struct curr_tss_desc, next_tss_desc;
2839 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2840 ulong old_tss_base =
2841 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2845 /* FIXME: old_tss_base == ~0 ? */
2847 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2848 if (ret != X86EMUL_CONTINUE)
2850 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2851 if (ret != X86EMUL_CONTINUE)
2854 /* FIXME: check that next_tss_desc is tss */
2857 * Check privileges. The three cases are task switch caused by...
2859 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2860 * 2. Exception/IRQ/iret: No check is performed
2861 * 3. jmp/call to TSS: Check against DPL of the TSS
2863 if (reason == TASK_SWITCH_GATE) {
2864 if (idt_index != -1) {
2865 /* Software interrupts */
2866 struct desc_struct task_gate_desc;
2869 ret = read_interrupt_descriptor(ctxt, idt_index,
2871 if (ret != X86EMUL_CONTINUE)
2874 dpl = task_gate_desc.dpl;
2875 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2876 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2878 } else if (reason != TASK_SWITCH_IRET) {
2879 int dpl = next_tss_desc.dpl;
2880 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2881 return emulate_gp(ctxt, tss_selector);
2885 desc_limit = desc_limit_scaled(&next_tss_desc);
2886 if (!next_tss_desc.p ||
2887 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2888 desc_limit < 0x2b)) {
2889 emulate_ts(ctxt, tss_selector & 0xfffc);
2890 return X86EMUL_PROPAGATE_FAULT;
2893 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2894 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2895 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2898 if (reason == TASK_SWITCH_IRET)
2899 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2901 /* set back link to prev task only if NT bit is set in eflags
2902 note that old_tss_sel is not used after this point */
2903 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2904 old_tss_sel = 0xffff;
2906 if (next_tss_desc.type & 8)
2907 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2908 old_tss_base, &next_tss_desc);
2910 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2911 old_tss_base, &next_tss_desc);
2912 if (ret != X86EMUL_CONTINUE)
2915 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2916 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2918 if (reason != TASK_SWITCH_IRET) {
2919 next_tss_desc.type |= (1 << 1); /* set busy flag */
2920 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2923 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2924 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2926 if (has_error_code) {
2927 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2928 ctxt->lock_prefix = 0;
2929 ctxt->src.val = (unsigned long) error_code;
2930 ret = em_push(ctxt);
2936 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2937 u16 tss_selector, int idt_index, int reason,
2938 bool has_error_code, u32 error_code)
2942 invalidate_registers(ctxt);
2943 ctxt->_eip = ctxt->eip;
2944 ctxt->dst.type = OP_NONE;
2946 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2947 has_error_code, error_code);
2949 if (rc == X86EMUL_CONTINUE) {
2950 ctxt->eip = ctxt->_eip;
2951 writeback_registers(ctxt);
2954 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2957 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2960 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2962 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2963 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2966 static int em_das(struct x86_emulate_ctxt *ctxt)
2969 bool af, cf, old_cf;
2971 cf = ctxt->eflags & X86_EFLAGS_CF;
2977 af = ctxt->eflags & X86_EFLAGS_AF;
2978 if ((al & 0x0f) > 9 || af) {
2980 cf = old_cf | (al >= 250);
2985 if (old_al > 0x99 || old_cf) {
2991 /* Set PF, ZF, SF */
2992 ctxt->src.type = OP_IMM;
2994 ctxt->src.bytes = 1;
2995 fastop(ctxt, em_or);
2996 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2998 ctxt->eflags |= X86_EFLAGS_CF;
3000 ctxt->eflags |= X86_EFLAGS_AF;
3001 return X86EMUL_CONTINUE;
3004 static int em_aam(struct x86_emulate_ctxt *ctxt)
3008 if (ctxt->src.val == 0)
3009 return emulate_de(ctxt);
3011 al = ctxt->dst.val & 0xff;
3012 ah = al / ctxt->src.val;
3013 al %= ctxt->src.val;
3015 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3017 /* Set PF, ZF, SF */
3018 ctxt->src.type = OP_IMM;
3020 ctxt->src.bytes = 1;
3021 fastop(ctxt, em_or);
3023 return X86EMUL_CONTINUE;
3026 static int em_aad(struct x86_emulate_ctxt *ctxt)
3028 u8 al = ctxt->dst.val & 0xff;
3029 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3031 al = (al + (ah * ctxt->src.val)) & 0xff;
3033 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3035 /* Set PF, ZF, SF */
3036 ctxt->src.type = OP_IMM;
3038 ctxt->src.bytes = 1;
3039 fastop(ctxt, em_or);
3041 return X86EMUL_CONTINUE;
3044 static int em_call(struct x86_emulate_ctxt *ctxt)
3046 long rel = ctxt->src.val;
3048 ctxt->src.val = (unsigned long)ctxt->_eip;
3050 return em_push(ctxt);
3053 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3059 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3060 old_eip = ctxt->_eip;
3062 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3063 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3064 return X86EMUL_CONTINUE;
3067 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3069 ctxt->src.val = old_cs;
3071 if (rc != X86EMUL_CONTINUE)
3074 ctxt->src.val = old_eip;
3075 return em_push(ctxt);
3078 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3082 ctxt->dst.type = OP_REG;
3083 ctxt->dst.addr.reg = &ctxt->_eip;
3084 ctxt->dst.bytes = ctxt->op_bytes;
3085 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3086 if (rc != X86EMUL_CONTINUE)
3088 rsp_increment(ctxt, ctxt->src.val);
3089 return X86EMUL_CONTINUE;
3092 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3094 /* Write back the register source. */
3095 ctxt->src.val = ctxt->dst.val;
3096 write_register_operand(&ctxt->src);
3098 /* Write back the memory destination with implicit LOCK prefix. */
3099 ctxt->dst.val = ctxt->src.orig_val;
3100 ctxt->lock_prefix = 1;
3101 return X86EMUL_CONTINUE;
3104 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3106 ctxt->dst.val = ctxt->src2.val;
3107 return fastop(ctxt, em_imul);
3110 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3112 ctxt->dst.type = OP_REG;
3113 ctxt->dst.bytes = ctxt->src.bytes;
3114 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3115 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3117 return X86EMUL_CONTINUE;
3120 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3124 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3125 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3126 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3127 return X86EMUL_CONTINUE;
3130 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3134 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3135 return emulate_gp(ctxt, 0);
3136 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3137 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3138 return X86EMUL_CONTINUE;
3141 static int em_mov(struct x86_emulate_ctxt *ctxt)
3143 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3144 return X86EMUL_CONTINUE;
3147 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3149 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3150 return emulate_gp(ctxt, 0);
3152 /* Disable writeback. */
3153 ctxt->dst.type = OP_NONE;
3154 return X86EMUL_CONTINUE;
3157 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3161 if (ctxt->mode == X86EMUL_MODE_PROT64)
3162 val = ctxt->src.val & ~0ULL;
3164 val = ctxt->src.val & ~0U;
3166 /* #UD condition is already handled. */
3167 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3168 return emulate_gp(ctxt, 0);
3170 /* Disable writeback. */
3171 ctxt->dst.type = OP_NONE;
3172 return X86EMUL_CONTINUE;
3175 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3179 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3180 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3181 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3182 return emulate_gp(ctxt, 0);
3184 return X86EMUL_CONTINUE;
3187 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3191 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3192 return emulate_gp(ctxt, 0);
3194 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3195 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3196 return X86EMUL_CONTINUE;
3199 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3201 if (ctxt->modrm_reg > VCPU_SREG_GS)
3202 return emulate_ud(ctxt);
3204 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3205 return X86EMUL_CONTINUE;
3208 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3210 u16 sel = ctxt->src.val;
3212 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3213 return emulate_ud(ctxt);
3215 if (ctxt->modrm_reg == VCPU_SREG_SS)
3216 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3218 /* Disable writeback. */
3219 ctxt->dst.type = OP_NONE;
3220 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3223 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3225 u16 sel = ctxt->src.val;
3227 /* Disable writeback. */
3228 ctxt->dst.type = OP_NONE;
3229 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3232 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3234 u16 sel = ctxt->src.val;
3236 /* Disable writeback. */
3237 ctxt->dst.type = OP_NONE;
3238 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3241 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3246 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3247 if (rc == X86EMUL_CONTINUE)
3248 ctxt->ops->invlpg(ctxt, linear);
3249 /* Disable writeback. */
3250 ctxt->dst.type = OP_NONE;
3251 return X86EMUL_CONTINUE;
3254 static int em_clts(struct x86_emulate_ctxt *ctxt)
3258 cr0 = ctxt->ops->get_cr(ctxt, 0);
3260 ctxt->ops->set_cr(ctxt, 0, cr0);
3261 return X86EMUL_CONTINUE;
3264 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3268 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3269 return X86EMUL_UNHANDLEABLE;
3271 rc = ctxt->ops->fix_hypercall(ctxt);
3272 if (rc != X86EMUL_CONTINUE)
3275 /* Let the processor re-execute the fixed hypercall */
3276 ctxt->_eip = ctxt->eip;
3277 /* Disable writeback. */
3278 ctxt->dst.type = OP_NONE;
3279 return X86EMUL_CONTINUE;
3282 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3283 void (*get)(struct x86_emulate_ctxt *ctxt,
3284 struct desc_ptr *ptr))
3286 struct desc_ptr desc_ptr;
3288 if (ctxt->mode == X86EMUL_MODE_PROT64)
3290 get(ctxt, &desc_ptr);
3291 if (ctxt->op_bytes == 2) {
3293 desc_ptr.address &= 0x00ffffff;
3295 /* Disable writeback. */
3296 ctxt->dst.type = OP_NONE;
3297 return segmented_write(ctxt, ctxt->dst.addr.mem,
3298 &desc_ptr, 2 + ctxt->op_bytes);
3301 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3303 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3306 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3308 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3311 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3313 struct desc_ptr desc_ptr;
3316 if (ctxt->mode == X86EMUL_MODE_PROT64)
3318 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3319 &desc_ptr.size, &desc_ptr.address,
3321 if (rc != X86EMUL_CONTINUE)
3323 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3324 /* Disable writeback. */
3325 ctxt->dst.type = OP_NONE;
3326 return X86EMUL_CONTINUE;
3329 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3333 rc = ctxt->ops->fix_hypercall(ctxt);
3335 /* Disable writeback. */
3336 ctxt->dst.type = OP_NONE;
3340 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3342 struct desc_ptr desc_ptr;
3345 if (ctxt->mode == X86EMUL_MODE_PROT64)
3347 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3348 &desc_ptr.size, &desc_ptr.address,
3350 if (rc != X86EMUL_CONTINUE)
3352 ctxt->ops->set_idt(ctxt, &desc_ptr);
3353 /* Disable writeback. */
3354 ctxt->dst.type = OP_NONE;
3355 return X86EMUL_CONTINUE;
3358 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3360 ctxt->dst.bytes = 2;
3361 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3362 return X86EMUL_CONTINUE;
3365 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3367 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3368 | (ctxt->src.val & 0x0f));
3369 ctxt->dst.type = OP_NONE;
3370 return X86EMUL_CONTINUE;
3373 static int em_loop(struct x86_emulate_ctxt *ctxt)
3375 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3376 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3377 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3378 jmp_rel(ctxt, ctxt->src.val);
3380 return X86EMUL_CONTINUE;
3383 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3385 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3386 jmp_rel(ctxt, ctxt->src.val);
3388 return X86EMUL_CONTINUE;
3391 static int em_in(struct x86_emulate_ctxt *ctxt)
3393 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3395 return X86EMUL_IO_NEEDED;
3397 return X86EMUL_CONTINUE;
3400 static int em_out(struct x86_emulate_ctxt *ctxt)
3402 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3406 return X86EMUL_CONTINUE;
3409 static int em_cli(struct x86_emulate_ctxt *ctxt)
3411 if (emulator_bad_iopl(ctxt))
3412 return emulate_gp(ctxt, 0);
3414 ctxt->eflags &= ~X86_EFLAGS_IF;
3415 return X86EMUL_CONTINUE;
3418 static int em_sti(struct x86_emulate_ctxt *ctxt)
3420 if (emulator_bad_iopl(ctxt))
3421 return emulate_gp(ctxt, 0);
3423 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3424 ctxt->eflags |= X86_EFLAGS_IF;
3425 return X86EMUL_CONTINUE;
3428 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3430 u32 eax, ebx, ecx, edx;
3432 eax = reg_read(ctxt, VCPU_REGS_RAX);
3433 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3434 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3435 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3436 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3437 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3438 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3439 return X86EMUL_CONTINUE;
3442 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3444 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3445 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3446 return X86EMUL_CONTINUE;
3449 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3451 switch (ctxt->op_bytes) {
3452 #ifdef CONFIG_X86_64
3454 asm("bswap %0" : "+r"(ctxt->dst.val));
3458 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3461 return X86EMUL_CONTINUE;
3464 static bool valid_cr(int nr)
3476 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3478 if (!valid_cr(ctxt->modrm_reg))
3479 return emulate_ud(ctxt);
3481 return X86EMUL_CONTINUE;
3484 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3486 u64 new_val = ctxt->src.val64;
3487 int cr = ctxt->modrm_reg;
3490 static u64 cr_reserved_bits[] = {
3491 0xffffffff00000000ULL,
3492 0, 0, 0, /* CR3 checked later */
3499 return emulate_ud(ctxt);
3501 if (new_val & cr_reserved_bits[cr])
3502 return emulate_gp(ctxt, 0);
3507 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3508 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3509 return emulate_gp(ctxt, 0);
3511 cr4 = ctxt->ops->get_cr(ctxt, 4);
3512 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3514 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3515 !(cr4 & X86_CR4_PAE))
3516 return emulate_gp(ctxt, 0);
3523 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3524 if (efer & EFER_LMA)
3525 rsvd = CR3_L_MODE_RESERVED_BITS;
3526 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3527 rsvd = CR3_PAE_RESERVED_BITS;
3528 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3529 rsvd = CR3_NONPAE_RESERVED_BITS;
3532 return emulate_gp(ctxt, 0);
3537 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3539 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3540 return emulate_gp(ctxt, 0);
3546 return X86EMUL_CONTINUE;
3549 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3553 ctxt->ops->get_dr(ctxt, 7, &dr7);
3555 /* Check if DR7.Global_Enable is set */
3556 return dr7 & (1 << 13);
3559 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3561 int dr = ctxt->modrm_reg;
3565 return emulate_ud(ctxt);
3567 cr4 = ctxt->ops->get_cr(ctxt, 4);
3568 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3569 return emulate_ud(ctxt);
3571 if (check_dr7_gd(ctxt))
3572 return emulate_db(ctxt);
3574 return X86EMUL_CONTINUE;
3577 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3579 u64 new_val = ctxt->src.val64;
3580 int dr = ctxt->modrm_reg;
3582 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3583 return emulate_gp(ctxt, 0);
3585 return check_dr_read(ctxt);
3588 static int check_svme(struct x86_emulate_ctxt *ctxt)
3592 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3594 if (!(efer & EFER_SVME))
3595 return emulate_ud(ctxt);
3597 return X86EMUL_CONTINUE;
3600 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3602 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3604 /* Valid physical address? */
3605 if (rax & 0xffff000000000000ULL)
3606 return emulate_gp(ctxt, 0);
3608 return check_svme(ctxt);
3611 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3613 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3615 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3616 return emulate_ud(ctxt);
3618 return X86EMUL_CONTINUE;
3621 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3623 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3624 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3626 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3628 return emulate_gp(ctxt, 0);
3630 return X86EMUL_CONTINUE;
3633 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3635 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3636 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3637 return emulate_gp(ctxt, 0);
3639 return X86EMUL_CONTINUE;
3642 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3644 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3645 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3646 return emulate_gp(ctxt, 0);
3648 return X86EMUL_CONTINUE;
3651 #define D(_y) { .flags = (_y) }
3652 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3653 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3654 .check_perm = (_p) }
3655 #define N D(NotImpl)
3656 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3657 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3658 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3659 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3660 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3661 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3662 #define II(_f, _e, _i) \
3663 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3664 #define IIP(_f, _e, _i, _p) \
3665 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3666 .check_perm = (_p) }
3667 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3669 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3670 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3671 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3672 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3673 #define I2bvIP(_f, _e, _i, _p) \
3674 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3676 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3677 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3678 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3680 static const struct opcode group7_rm1[] = {
3681 DI(SrcNone | Priv, monitor),
3682 DI(SrcNone | Priv, mwait),
3686 static const struct opcode group7_rm3[] = {
3687 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3688 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3689 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3690 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3691 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3692 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3693 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3694 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3697 static const struct opcode group7_rm7[] = {
3699 DIP(SrcNone, rdtscp, check_rdtsc),
3703 static const struct opcode group1[] = {
3705 F(Lock | PageTable, em_or),
3708 F(Lock | PageTable, em_and),
3714 static const struct opcode group1A[] = {
3715 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3718 static const struct opcode group2[] = {
3719 F(DstMem | ModRM, em_rol),
3720 F(DstMem | ModRM, em_ror),
3721 F(DstMem | ModRM, em_rcl),
3722 F(DstMem | ModRM, em_rcr),
3723 F(DstMem | ModRM, em_shl),
3724 F(DstMem | ModRM, em_shr),
3725 F(DstMem | ModRM, em_shl),
3726 F(DstMem | ModRM, em_sar),
3729 static const struct opcode group3[] = {
3730 F(DstMem | SrcImm | NoWrite, em_test),
3731 F(DstMem | SrcImm | NoWrite, em_test),
3732 F(DstMem | SrcNone | Lock, em_not),
3733 F(DstMem | SrcNone | Lock, em_neg),
3734 I(SrcMem, em_mul_ex),
3735 I(SrcMem, em_imul_ex),
3736 I(SrcMem, em_div_ex),
3737 I(SrcMem, em_idiv_ex),
3740 static const struct opcode group4[] = {
3741 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3742 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3746 static const struct opcode group5[] = {
3747 F(DstMem | SrcNone | Lock, em_inc),
3748 F(DstMem | SrcNone | Lock, em_dec),
3749 I(SrcMem | Stack, em_grp45),
3750 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3751 I(SrcMem | Stack, em_grp45),
3752 I(SrcMemFAddr | ImplicitOps, em_grp45),
3753 I(SrcMem | Stack, em_grp45), D(Undefined),
3756 static const struct opcode group6[] = {
3759 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3760 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3764 static const struct group_dual group7 = { {
3765 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3766 II(Mov | DstMem | Priv, em_sidt, sidt),
3767 II(SrcMem | Priv, em_lgdt, lgdt),
3768 II(SrcMem | Priv, em_lidt, lidt),
3769 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3770 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3771 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3773 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3775 N, EXT(0, group7_rm3),
3776 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3777 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3781 static const struct opcode group8[] = {
3783 F(DstMem | SrcImmByte | NoWrite, em_bt),
3784 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3785 F(DstMem | SrcImmByte | Lock, em_btr),
3786 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3789 static const struct group_dual group9 = { {
3790 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3792 N, N, N, N, N, N, N, N,
3795 static const struct opcode group11[] = {
3796 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3800 static const struct gprefix pfx_0f_6f_0f_7f = {
3801 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3804 static const struct gprefix pfx_vmovntpx = {
3805 I(0, em_mov), N, N, N,
3808 static const struct escape escape_d9 = { {
3809 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3812 N, N, N, N, N, N, N, N,
3814 N, N, N, N, N, N, N, N,
3816 N, N, N, N, N, N, N, N,
3818 N, N, N, N, N, N, N, N,
3820 N, N, N, N, N, N, N, N,
3822 N, N, N, N, N, N, N, N,
3824 N, N, N, N, N, N, N, N,
3826 N, N, N, N, N, N, N, N,
3829 static const struct escape escape_db = { {
3830 N, N, N, N, N, N, N, N,
3833 N, N, N, N, N, N, N, N,
3835 N, N, N, N, N, N, N, N,
3837 N, N, N, N, N, N, N, N,
3839 N, N, N, N, N, N, N, N,
3841 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3843 N, N, N, N, N, N, N, N,
3845 N, N, N, N, N, N, N, N,
3847 N, N, N, N, N, N, N, N,
3850 static const struct escape escape_dd = { {
3851 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3854 N, N, N, N, N, N, N, N,
3856 N, N, N, N, N, N, N, N,
3858 N, N, N, N, N, N, N, N,
3860 N, N, N, N, N, N, N, N,
3862 N, N, N, N, N, N, N, N,
3864 N, N, N, N, N, N, N, N,
3866 N, N, N, N, N, N, N, N,
3868 N, N, N, N, N, N, N, N,
3871 static const struct opcode opcode_table[256] = {
3873 F6ALU(Lock, em_add),
3874 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3875 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3877 F6ALU(Lock | PageTable, em_or),
3878 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3881 F6ALU(Lock, em_adc),
3882 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3883 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3885 F6ALU(Lock, em_sbb),
3886 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3887 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3889 F6ALU(Lock | PageTable, em_and), N, N,
3891 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3893 F6ALU(Lock, em_xor), N, N,
3895 F6ALU(NoWrite, em_cmp), N, N,
3897 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3899 X8(I(SrcReg | Stack, em_push)),
3901 X8(I(DstReg | Stack, em_pop)),
3903 I(ImplicitOps | Stack | No64, em_pusha),
3904 I(ImplicitOps | Stack | No64, em_popa),
3905 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3908 I(SrcImm | Mov | Stack, em_push),
3909 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3910 I(SrcImmByte | Mov | Stack, em_push),
3911 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3912 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3913 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3917 G(ByteOp | DstMem | SrcImm, group1),
3918 G(DstMem | SrcImm, group1),
3919 G(ByteOp | DstMem | SrcImm | No64, group1),
3920 G(DstMem | SrcImmByte, group1),
3921 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3922 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3924 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3925 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3926 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3927 D(ModRM | SrcMem | NoAccess | DstReg),
3928 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3931 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3933 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3934 I(SrcImmFAddr | No64, em_call_far), N,
3935 II(ImplicitOps | Stack, em_pushf, pushf),
3936 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3938 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3939 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3940 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3941 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3943 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3944 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3945 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3946 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3948 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3950 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3952 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3953 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3954 I(ImplicitOps | Stack, em_ret),
3955 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3956 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3957 G(ByteOp, group11), G(0, group11),
3959 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3960 N, I(ImplicitOps | Stack, em_ret_far),
3961 D(ImplicitOps), DI(SrcImmByte, intn),
3962 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3964 G(Src2One | ByteOp, group2), G(Src2One, group2),
3965 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3966 I(DstAcc | SrcImmUByte | No64, em_aam),
3967 I(DstAcc | SrcImmUByte | No64, em_aad),
3968 F(DstAcc | ByteOp | No64, em_salc),
3969 I(DstAcc | SrcXLat | ByteOp, em_mov),
3971 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3973 X3(I(SrcImmByte, em_loop)),
3974 I(SrcImmByte, em_jcxz),
3975 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3976 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3978 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3979 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3980 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3981 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3983 N, DI(ImplicitOps, icebp), N, N,
3984 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3985 G(ByteOp, group3), G(0, group3),
3987 D(ImplicitOps), D(ImplicitOps),
3988 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3989 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3992 static const struct opcode twobyte_table[256] = {
3994 G(0, group6), GD(0, &group7), N, N,
3995 N, I(ImplicitOps | VendorSpecific, em_syscall),
3996 II(ImplicitOps | Priv, em_clts, clts), N,
3997 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3998 N, D(ImplicitOps | ModRM), N, N,
4000 N, N, N, N, N, N, N, N,
4001 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
4003 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4004 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4005 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4006 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4008 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4011 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4012 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4013 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4014 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4015 I(ImplicitOps | VendorSpecific, em_sysenter),
4016 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4018 N, N, N, N, N, N, N, N,
4020 X16(D(DstReg | SrcMem | ModRM | Mov)),
4022 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4027 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4032 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4036 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4038 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4039 II(ImplicitOps, em_cpuid, cpuid),
4040 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4041 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4042 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4044 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4045 DI(ImplicitOps, rsm),
4046 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4047 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4048 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4049 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4051 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4052 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4053 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4054 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4055 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4056 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4060 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4061 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4062 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4064 D2bv(DstMem | SrcReg | ModRM | Lock),
4065 N, D(DstMem | SrcReg | ModRM | Mov),
4066 N, N, N, GD(0, &group9),
4068 X8(I(DstReg, em_bswap)),
4070 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4072 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4074 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4091 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4095 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4101 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4102 unsigned size, bool sign_extension)
4104 int rc = X86EMUL_CONTINUE;
4108 op->addr.mem.ea = ctxt->_eip;
4109 /* NB. Immediates are sign-extended as necessary. */
4110 switch (op->bytes) {
4112 op->val = insn_fetch(s8, ctxt);
4115 op->val = insn_fetch(s16, ctxt);
4118 op->val = insn_fetch(s32, ctxt);
4121 op->val = insn_fetch(s64, ctxt);
4124 if (!sign_extension) {
4125 switch (op->bytes) {
4133 op->val &= 0xffffffff;
4141 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4144 int rc = X86EMUL_CONTINUE;
4148 decode_register_operand(ctxt, op);
4151 rc = decode_imm(ctxt, op, 1, false);
4154 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4158 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4159 fetch_bit_operand(ctxt);
4160 op->orig_val = op->val;
4163 ctxt->memop.bytes = 8;
4167 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4168 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4169 fetch_register_operand(op);
4170 op->orig_val = op->val;
4174 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4177 op->addr.mem.seg = VCPU_SREG_ES;
4184 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4185 fetch_register_operand(op);
4189 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4192 rc = decode_imm(ctxt, op, 1, true);
4199 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4202 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4205 ctxt->memop.bytes = 1;
4206 if (ctxt->memop.type == OP_REG) {
4207 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4208 fetch_register_operand(&ctxt->memop);
4212 ctxt->memop.bytes = 2;
4215 ctxt->memop.bytes = 4;
4218 rc = decode_imm(ctxt, op, 2, false);
4221 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4225 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4227 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4228 op->addr.mem.seg = seg_override(ctxt);
4234 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4236 register_address(ctxt,
4237 reg_read(ctxt, VCPU_REGS_RBX) +
4238 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4239 op->addr.mem.seg = seg_override(ctxt);
4244 op->addr.mem.ea = ctxt->_eip;
4245 op->bytes = ctxt->op_bytes + 2;
4246 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4249 ctxt->memop.bytes = ctxt->op_bytes + 2;
4252 op->val = VCPU_SREG_ES;
4255 op->val = VCPU_SREG_CS;
4258 op->val = VCPU_SREG_SS;
4261 op->val = VCPU_SREG_DS;
4264 op->val = VCPU_SREG_FS;
4267 op->val = VCPU_SREG_GS;
4270 /* Special instructions do their own operand decoding. */
4272 op->type = OP_NONE; /* Disable writeback. */
4280 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4282 int rc = X86EMUL_CONTINUE;
4283 int mode = ctxt->mode;
4284 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4285 bool op_prefix = false;
4286 struct opcode opcode;
4288 ctxt->memop.type = OP_NONE;
4289 ctxt->memopp = NULL;
4290 ctxt->_eip = ctxt->eip;
4291 ctxt->fetch.start = ctxt->_eip;
4292 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4294 memcpy(ctxt->fetch.data, insn, insn_len);
4297 case X86EMUL_MODE_REAL:
4298 case X86EMUL_MODE_VM86:
4299 case X86EMUL_MODE_PROT16:
4300 def_op_bytes = def_ad_bytes = 2;
4302 case X86EMUL_MODE_PROT32:
4303 def_op_bytes = def_ad_bytes = 4;
4305 #ifdef CONFIG_X86_64
4306 case X86EMUL_MODE_PROT64:
4312 return EMULATION_FAILED;
4315 ctxt->op_bytes = def_op_bytes;
4316 ctxt->ad_bytes = def_ad_bytes;
4318 /* Legacy prefixes. */
4320 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4321 case 0x66: /* operand-size override */
4323 /* switch between 2/4 bytes */
4324 ctxt->op_bytes = def_op_bytes ^ 6;
4326 case 0x67: /* address-size override */
4327 if (mode == X86EMUL_MODE_PROT64)
4328 /* switch between 4/8 bytes */
4329 ctxt->ad_bytes = def_ad_bytes ^ 12;
4331 /* switch between 2/4 bytes */
4332 ctxt->ad_bytes = def_ad_bytes ^ 6;
4334 case 0x26: /* ES override */
4335 case 0x2e: /* CS override */
4336 case 0x36: /* SS override */
4337 case 0x3e: /* DS override */
4338 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4340 case 0x64: /* FS override */
4341 case 0x65: /* GS override */
4342 set_seg_override(ctxt, ctxt->b & 7);
4344 case 0x40 ... 0x4f: /* REX */
4345 if (mode != X86EMUL_MODE_PROT64)
4347 ctxt->rex_prefix = ctxt->b;
4349 case 0xf0: /* LOCK */
4350 ctxt->lock_prefix = 1;
4352 case 0xf2: /* REPNE/REPNZ */
4353 case 0xf3: /* REP/REPE/REPZ */
4354 ctxt->rep_prefix = ctxt->b;
4360 /* Any legacy prefix after a REX prefix nullifies its effect. */
4362 ctxt->rex_prefix = 0;
4368 if (ctxt->rex_prefix & 8)
4369 ctxt->op_bytes = 8; /* REX.W */
4371 /* Opcode byte(s). */
4372 opcode = opcode_table[ctxt->b];
4373 /* Two-byte opcode? */
4374 if (ctxt->b == 0x0f) {
4376 ctxt->b = insn_fetch(u8, ctxt);
4377 opcode = twobyte_table[ctxt->b];
4379 ctxt->d = opcode.flags;
4381 if (ctxt->d & ModRM)
4382 ctxt->modrm = insn_fetch(u8, ctxt);
4384 while (ctxt->d & GroupMask) {
4385 switch (ctxt->d & GroupMask) {
4387 goffset = (ctxt->modrm >> 3) & 7;
4388 opcode = opcode.u.group[goffset];
4391 goffset = (ctxt->modrm >> 3) & 7;
4392 if ((ctxt->modrm >> 6) == 3)
4393 opcode = opcode.u.gdual->mod3[goffset];
4395 opcode = opcode.u.gdual->mod012[goffset];
4398 goffset = ctxt->modrm & 7;
4399 opcode = opcode.u.group[goffset];
4402 if (ctxt->rep_prefix && op_prefix)
4403 return EMULATION_FAILED;
4404 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4405 switch (simd_prefix) {
4406 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4407 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4408 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4409 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4413 if (ctxt->modrm > 0xbf)
4414 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4416 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4419 return EMULATION_FAILED;
4422 ctxt->d &= ~(u64)GroupMask;
4423 ctxt->d |= opcode.flags;
4426 ctxt->execute = opcode.u.execute;
4427 ctxt->check_perm = opcode.check_perm;
4428 ctxt->intercept = opcode.intercept;
4431 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4432 return EMULATION_FAILED;
4434 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4435 return EMULATION_FAILED;
4437 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4440 if (ctxt->d & Op3264) {
4441 if (mode == X86EMUL_MODE_PROT64)
4448 ctxt->op_bytes = 16;
4449 else if (ctxt->d & Mmx)
4452 /* ModRM and SIB bytes. */
4453 if (ctxt->d & ModRM) {
4454 rc = decode_modrm(ctxt, &ctxt->memop);
4455 if (!ctxt->has_seg_override)
4456 set_seg_override(ctxt, ctxt->modrm_seg);
4457 } else if (ctxt->d & MemAbs)
4458 rc = decode_abs(ctxt, &ctxt->memop);
4459 if (rc != X86EMUL_CONTINUE)
4462 if (!ctxt->has_seg_override)
4463 set_seg_override(ctxt, VCPU_SREG_DS);
4465 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4467 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4468 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4471 * Decode and fetch the source operand: register, memory
4474 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4475 if (rc != X86EMUL_CONTINUE)
4479 * Decode and fetch the second source operand: register, memory
4482 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4483 if (rc != X86EMUL_CONTINUE)
4486 /* Decode and fetch the destination operand: register or memory. */
4487 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4490 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4491 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4493 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4496 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4498 return ctxt->d & PageTable;
4501 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4503 /* The second termination condition only applies for REPE
4504 * and REPNE. Test if the repeat string operation prefix is
4505 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4506 * corresponding termination condition according to:
4507 * - if REPE/REPZ and ZF = 0 then done
4508 * - if REPNE/REPNZ and ZF = 1 then done
4510 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4511 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4512 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4513 ((ctxt->eflags & EFLG_ZF) == 0))
4514 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4515 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4521 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4525 ctxt->ops->get_fpu(ctxt);
4526 asm volatile("1: fwait \n\t"
4528 ".pushsection .fixup,\"ax\" \n\t"
4530 "movb $1, %[fault] \n\t"
4533 _ASM_EXTABLE(1b, 3b)
4534 : [fault]"+qm"(fault));
4535 ctxt->ops->put_fpu(ctxt);
4537 if (unlikely(fault))
4538 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4540 return X86EMUL_CONTINUE;
4543 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4546 if (op->type == OP_MM)
4547 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4550 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4552 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4553 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4554 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4555 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4556 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4557 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4558 return X86EMUL_CONTINUE;
4561 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4563 const struct x86_emulate_ops *ops = ctxt->ops;
4564 int rc = X86EMUL_CONTINUE;
4565 int saved_dst_type = ctxt->dst.type;
4567 ctxt->mem_read.pos = 0;
4569 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4570 (ctxt->d & Undefined)) {
4571 rc = emulate_ud(ctxt);
4575 /* LOCK prefix is allowed only with some instructions */
4576 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4577 rc = emulate_ud(ctxt);
4581 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4582 rc = emulate_ud(ctxt);
4586 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4587 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4588 rc = emulate_ud(ctxt);
4592 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4593 rc = emulate_nm(ctxt);
4597 if (ctxt->d & Mmx) {
4598 rc = flush_pending_x87_faults(ctxt);
4599 if (rc != X86EMUL_CONTINUE)
4602 * Now that we know the fpu is exception safe, we can fetch
4605 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4606 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4607 if (!(ctxt->d & Mov))
4608 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4611 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4612 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4613 X86_ICPT_PRE_EXCEPT);
4614 if (rc != X86EMUL_CONTINUE)
4618 /* Privileged instruction can be executed only in CPL=0 */
4619 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4620 rc = emulate_gp(ctxt, 0);
4624 /* Instruction can only be executed in protected mode */
4625 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4626 rc = emulate_ud(ctxt);
4630 /* Do instruction specific permission checks */
4631 if (ctxt->check_perm) {
4632 rc = ctxt->check_perm(ctxt);
4633 if (rc != X86EMUL_CONTINUE)
4637 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4638 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4639 X86_ICPT_POST_EXCEPT);
4640 if (rc != X86EMUL_CONTINUE)
4644 if (ctxt->rep_prefix && (ctxt->d & String)) {
4645 /* All REP prefixes have the same first termination condition */
4646 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4647 ctxt->eip = ctxt->_eip;
4652 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4653 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4654 ctxt->src.valptr, ctxt->src.bytes);
4655 if (rc != X86EMUL_CONTINUE)
4657 ctxt->src.orig_val64 = ctxt->src.val64;
4660 if (ctxt->src2.type == OP_MEM) {
4661 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4662 &ctxt->src2.val, ctxt->src2.bytes);
4663 if (rc != X86EMUL_CONTINUE)
4667 if ((ctxt->d & DstMask) == ImplicitOps)
4671 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4672 /* optimisation - avoid slow emulated read if Mov */
4673 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4674 &ctxt->dst.val, ctxt->dst.bytes);
4675 if (rc != X86EMUL_CONTINUE)
4678 ctxt->dst.orig_val = ctxt->dst.val;
4682 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4683 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4684 X86_ICPT_POST_MEMACCESS);
4685 if (rc != X86EMUL_CONTINUE)
4689 if (ctxt->execute) {
4690 if (ctxt->d & Fastop) {
4691 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4692 rc = fastop(ctxt, fop);
4693 if (rc != X86EMUL_CONTINUE)
4697 rc = ctxt->execute(ctxt);
4698 if (rc != X86EMUL_CONTINUE)
4707 case 0x63: /* movsxd */
4708 if (ctxt->mode != X86EMUL_MODE_PROT64)
4709 goto cannot_emulate;
4710 ctxt->dst.val = (s32) ctxt->src.val;
4712 case 0x70 ... 0x7f: /* jcc (short) */
4713 if (test_cc(ctxt->b, ctxt->eflags))
4714 jmp_rel(ctxt, ctxt->src.val);
4716 case 0x8d: /* lea r16/r32, m */
4717 ctxt->dst.val = ctxt->src.addr.mem.ea;
4719 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4720 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4724 case 0x98: /* cbw/cwde/cdqe */
4725 switch (ctxt->op_bytes) {
4726 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4727 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4728 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4731 case 0xcc: /* int3 */
4732 rc = emulate_int(ctxt, 3);
4734 case 0xcd: /* int n */
4735 rc = emulate_int(ctxt, ctxt->src.val);
4737 case 0xce: /* into */
4738 if (ctxt->eflags & EFLG_OF)
4739 rc = emulate_int(ctxt, 4);
4741 case 0xe9: /* jmp rel */
4742 case 0xeb: /* jmp rel short */
4743 jmp_rel(ctxt, ctxt->src.val);
4744 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4746 case 0xf4: /* hlt */
4747 ctxt->ops->halt(ctxt);
4749 case 0xf5: /* cmc */
4750 /* complement carry flag from eflags reg */
4751 ctxt->eflags ^= EFLG_CF;
4753 case 0xf8: /* clc */
4754 ctxt->eflags &= ~EFLG_CF;
4756 case 0xf9: /* stc */
4757 ctxt->eflags |= EFLG_CF;
4759 case 0xfc: /* cld */
4760 ctxt->eflags &= ~EFLG_DF;
4762 case 0xfd: /* std */
4763 ctxt->eflags |= EFLG_DF;
4766 goto cannot_emulate;
4769 if (rc != X86EMUL_CONTINUE)
4773 rc = writeback(ctxt);
4774 if (rc != X86EMUL_CONTINUE)
4778 * restore dst type in case the decoding will be reused
4779 * (happens for string instruction )
4781 ctxt->dst.type = saved_dst_type;
4783 if ((ctxt->d & SrcMask) == SrcSI)
4784 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4786 if ((ctxt->d & DstMask) == DstDI)
4787 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4789 if (ctxt->rep_prefix && (ctxt->d & String)) {
4791 struct read_cache *r = &ctxt->io_read;
4792 if ((ctxt->d & SrcMask) == SrcSI)
4793 count = ctxt->src.count;
4795 count = ctxt->dst.count;
4796 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4799 if (!string_insn_completed(ctxt)) {
4801 * Re-enter guest when pio read ahead buffer is empty
4802 * or, if it is not used, after each 1024 iteration.
4804 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4805 (r->end == 0 || r->end != r->pos)) {
4807 * Reset read cache. Usually happens before
4808 * decode, but since instruction is restarted
4809 * we have to do it here.
4811 ctxt->mem_read.end = 0;
4812 writeback_registers(ctxt);
4813 return EMULATION_RESTART;
4815 goto done; /* skip rip writeback */
4819 ctxt->eip = ctxt->_eip;
4822 if (rc == X86EMUL_PROPAGATE_FAULT)
4823 ctxt->have_exception = true;
4824 if (rc == X86EMUL_INTERCEPTED)
4825 return EMULATION_INTERCEPTED;
4827 if (rc == X86EMUL_CONTINUE)
4828 writeback_registers(ctxt);
4830 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4834 case 0x09: /* wbinvd */
4835 (ctxt->ops->wbinvd)(ctxt);
4837 case 0x08: /* invd */
4838 case 0x0d: /* GrpP (prefetch) */
4839 case 0x18: /* Grp16 (prefetch/nop) */
4840 case 0x1f: /* nop */
4842 case 0x20: /* mov cr, reg */
4843 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4845 case 0x21: /* mov from dr to reg */
4846 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4848 case 0x40 ... 0x4f: /* cmov */
4849 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4850 if (!test_cc(ctxt->b, ctxt->eflags))
4851 ctxt->dst.type = OP_NONE; /* no writeback */
4853 case 0x80 ... 0x8f: /* jnz rel, etc*/
4854 if (test_cc(ctxt->b, ctxt->eflags))
4855 jmp_rel(ctxt, ctxt->src.val);
4857 case 0x90 ... 0x9f: /* setcc r/m8 */
4858 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4860 case 0xae: /* clflush */
4862 case 0xb6 ... 0xb7: /* movzx */
4863 ctxt->dst.bytes = ctxt->op_bytes;
4864 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4865 : (u16) ctxt->src.val;
4867 case 0xbe ... 0xbf: /* movsx */
4868 ctxt->dst.bytes = ctxt->op_bytes;
4869 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4870 (s16) ctxt->src.val;
4872 case 0xc0 ... 0xc1: /* xadd */
4873 fastop(ctxt, em_add);
4874 /* Write back the register source. */
4875 ctxt->src.val = ctxt->dst.orig_val;
4876 write_register_operand(&ctxt->src);
4878 case 0xc3: /* movnti */
4879 ctxt->dst.bytes = ctxt->op_bytes;
4880 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4881 (u64) ctxt->src.val;
4884 goto cannot_emulate;
4887 if (rc != X86EMUL_CONTINUE)
4893 return EMULATION_FAILED;
4896 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4898 invalidate_registers(ctxt);
4901 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4903 writeback_registers(ctxt);