1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
167 #define NoBigReal ((u64)1 << 50) /* No big real mode */
168 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
170 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
172 #define X2(x...) x, x
173 #define X3(x...) X2(x), x
174 #define X4(x...) X2(x), X2(x)
175 #define X5(x...) X4(x), x
176 #define X6(x...) X4(x), X2(x)
177 #define X7(x...) X4(x), X3(x)
178 #define X8(x...) X4(x), X4(x)
179 #define X16(x...) X8(x), X8(x)
181 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
182 #define FASTOP_SIZE 8
185 * fastop functions have a special calling convention:
190 * flags: rflags (in/out)
191 * ex: rsi (in:fastop pointer, out:zero if exception)
193 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
194 * different operand sizes can be reached by calculation, rather than a jump
195 * table (which would be bigger than the code).
197 * fastop functions are declared as taking a never-defined fastop parameter,
198 * so they can't be called from C directly.
207 int (*execute)(struct x86_emulate_ctxt *ctxt);
208 const struct opcode *group;
209 const struct group_dual *gdual;
210 const struct gprefix *gprefix;
211 const struct escape *esc;
212 void (*fastop)(struct fastop *fake);
214 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
218 struct opcode mod012[8];
219 struct opcode mod3[8];
223 struct opcode pfx_no;
224 struct opcode pfx_66;
225 struct opcode pfx_f2;
226 struct opcode pfx_f3;
231 struct opcode high[64];
234 /* EFLAGS bit definitions. */
235 #define EFLG_ID (1<<21)
236 #define EFLG_VIP (1<<20)
237 #define EFLG_VIF (1<<19)
238 #define EFLG_AC (1<<18)
239 #define EFLG_VM (1<<17)
240 #define EFLG_RF (1<<16)
241 #define EFLG_IOPL (3<<12)
242 #define EFLG_NT (1<<14)
243 #define EFLG_OF (1<<11)
244 #define EFLG_DF (1<<10)
245 #define EFLG_IF (1<<9)
246 #define EFLG_TF (1<<8)
247 #define EFLG_SF (1<<7)
248 #define EFLG_ZF (1<<6)
249 #define EFLG_AF (1<<4)
250 #define EFLG_PF (1<<2)
251 #define EFLG_CF (1<<0)
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 #define EFLG_RESERVED_ONE_MASK 2
256 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 if (!(ctxt->regs_valid & (1 << nr))) {
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
262 return ctxt->_regs[nr];
265 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->regs_dirty |= 1 << nr;
269 return &ctxt->_regs[nr];
272 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 return reg_write(ctxt, nr);
278 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
282 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
283 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
286 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
288 ctxt->regs_dirty = 0;
289 ctxt->regs_valid = 0;
293 * These EFLAGS bits are restored from saved value during emulation, and
294 * any changes are written back to the saved value after emulation.
296 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
304 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
306 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
307 #define FOP_RET "ret \n\t"
309 #define FOP_START(op) \
310 extern void em_##op(struct fastop *fake); \
311 asm(".pushsection .text, \"ax\" \n\t" \
312 ".global em_" #op " \n\t" \
319 #define FOPNOP() FOP_ALIGN FOP_RET
321 #define FOP1E(op, dst) \
322 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
324 #define FOP1EEX(op, dst) \
325 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
327 #define FASTOP1(op) \
332 ON64(FOP1E(op##q, rax)) \
335 /* 1-operand, using src2 (for MUL/DIV r/m) */
336 #define FASTOP1SRC2(op, name) \
341 ON64(FOP1E(op, rcx)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
345 #define FASTOP1SRC2EX(op, name) \
350 ON64(FOP1EEX(op, rcx)) \
353 #define FOP2E(op, dst, src) \
354 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
356 #define FASTOP2(op) \
358 FOP2E(op##b, al, dl) \
359 FOP2E(op##w, ax, dx) \
360 FOP2E(op##l, eax, edx) \
361 ON64(FOP2E(op##q, rax, rdx)) \
364 /* 2 operand, word only */
365 #define FASTOP2W(op) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, src is CL */
374 #define FASTOP2CL(op) \
376 FOP2E(op##b, al, cl) \
377 FOP2E(op##w, ax, cl) \
378 FOP2E(op##l, eax, cl) \
379 ON64(FOP2E(op##q, rax, cl)) \
382 #define FOP3E(op, dst, src, src2) \
383 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
385 /* 3-operand, word-only, src2=cl */
386 #define FASTOP3WCL(op) \
389 FOP3E(op##w, ax, dx, cl) \
390 FOP3E(op##l, eax, edx, cl) \
391 ON64(FOP3E(op##q, rax, rdx, cl)) \
394 /* Special case for SETcc - 1 instruction per cc */
395 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
397 asm(".global kvm_fastop_exception \n"
398 "kvm_fastop_exception: xor %esi, %esi; ret");
419 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
422 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
423 enum x86_intercept intercept,
424 enum x86_intercept_stage stage)
426 struct x86_instruction_info info = {
427 .intercept = intercept,
428 .rep_prefix = ctxt->rep_prefix,
429 .modrm_mod = ctxt->modrm_mod,
430 .modrm_reg = ctxt->modrm_reg,
431 .modrm_rm = ctxt->modrm_rm,
432 .src_val = ctxt->src.val64,
433 .dst_val = ctxt->dst.val64,
434 .src_bytes = ctxt->src.bytes,
435 .dst_bytes = ctxt->dst.bytes,
436 .ad_bytes = ctxt->ad_bytes,
437 .next_rip = ctxt->eip,
440 return ctxt->ops->intercept(ctxt, &info, stage);
443 static void assign_masked(ulong *dest, ulong src, ulong mask)
445 *dest = (*dest & ~mask) | (src & mask);
448 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
450 return (1UL << (ctxt->ad_bytes << 3)) - 1;
453 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
456 struct desc_struct ss;
458 if (ctxt->mode == X86EMUL_MODE_PROT64)
460 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
461 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
464 static int stack_size(struct x86_emulate_ctxt *ctxt)
466 return (__fls(stack_mask(ctxt)) + 1) >> 3;
469 /* Access/update address held in a register, based on addressing mode. */
470 static inline unsigned long
471 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
473 if (ctxt->ad_bytes == sizeof(unsigned long))
476 return reg & ad_mask(ctxt);
479 static inline unsigned long
480 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
482 return address_mask(ctxt, reg);
485 static void masked_increment(ulong *reg, ulong mask, int inc)
487 assign_masked(reg, *reg + inc, mask);
491 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
495 if (ctxt->ad_bytes == sizeof(unsigned long))
498 mask = ad_mask(ctxt);
499 masked_increment(reg, mask, inc);
502 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
504 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
516 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
519 return ctxt->ops->get_cached_segment_base(ctxt, seg);
522 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
523 u32 error, bool valid)
526 ctxt->exception.vector = vec;
527 ctxt->exception.error_code = error;
528 ctxt->exception.error_code_valid = valid;
529 return X86EMUL_PROPAGATE_FAULT;
532 static int emulate_db(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, DB_VECTOR, 0, false);
537 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
539 return emulate_exception(ctxt, GP_VECTOR, err, true);
542 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, SS_VECTOR, err, true);
547 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
549 return emulate_exception(ctxt, UD_VECTOR, 0, false);
552 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
554 return emulate_exception(ctxt, TS_VECTOR, err, true);
557 static int emulate_de(struct x86_emulate_ctxt *ctxt)
559 return emulate_exception(ctxt, DE_VECTOR, 0, false);
562 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
564 return emulate_exception(ctxt, NM_VECTOR, 0, false);
567 static inline int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
570 switch (ctxt->op_bytes) {
572 ctxt->_eip = (u16)dst;
575 ctxt->_eip = (u32)dst;
578 if ((cs_l && is_noncanonical_address(dst)) ||
579 (!cs_l && (dst & ~(u32)-1)))
580 return emulate_gp(ctxt, 0);
584 WARN(1, "unsupported eip assignment size\n");
586 return X86EMUL_CONTINUE;
589 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
591 return assign_eip_far(ctxt, dst, ctxt->mode == X86EMUL_MODE_PROT64);
594 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
596 return assign_eip_near(ctxt, ctxt->_eip + rel);
599 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
602 struct desc_struct desc;
604 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
608 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
613 struct desc_struct desc;
615 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
616 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
620 * x86 defines three classes of vector instructions: explicitly
621 * aligned, explicitly unaligned, and the rest, which change behaviour
622 * depending on whether they're AVX encoded or not.
624 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
625 * subject to the same check.
627 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
629 if (likely(size < 16))
632 if (ctxt->d & Aligned)
634 else if (ctxt->d & Unaligned)
636 else if (ctxt->d & Avx)
642 static int __linearize(struct x86_emulate_ctxt *ctxt,
643 struct segmented_address addr,
644 unsigned size, bool write, bool fetch,
647 struct desc_struct desc;
654 la = seg_base(ctxt, addr.seg) + addr.ea;
655 switch (ctxt->mode) {
656 case X86EMUL_MODE_PROT64:
657 if (((signed long)la << 16) >> 16 != la)
658 return emulate_gp(ctxt, 0);
661 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
665 /* code segment in protected mode or read-only data segment */
666 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
667 || !(desc.type & 2)) && write)
669 /* unreadable code segment */
670 if (!fetch && (desc.type & 8) && !(desc.type & 2))
672 lim = desc_limit_scaled(&desc);
673 if ((ctxt->mode == X86EMUL_MODE_REAL) && !fetch &&
674 (ctxt->d & NoBigReal)) {
675 /* la is between zero and 0xffff */
676 if (la > 0xffff || (u32)(la + size - 1) > 0xffff)
678 } else if ((desc.type & 8) || !(desc.type & 4)) {
679 /* expand-up segment */
680 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
683 /* expand-down segment */
684 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
686 lim = desc.d ? 0xffffffff : 0xffff;
687 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
690 cpl = ctxt->ops->cpl(ctxt);
691 if (!(desc.type & 8)) {
695 } else if ((desc.type & 8) && !(desc.type & 4)) {
696 /* nonconforming code segment */
699 } else if ((desc.type & 8) && (desc.type & 4)) {
700 /* conforming code segment */
706 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
708 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
709 return emulate_gp(ctxt, 0);
711 return X86EMUL_CONTINUE;
713 if (addr.seg == VCPU_SREG_SS)
714 return emulate_ss(ctxt, sel);
716 return emulate_gp(ctxt, sel);
719 static int linearize(struct x86_emulate_ctxt *ctxt,
720 struct segmented_address addr,
721 unsigned size, bool write,
724 return __linearize(ctxt, addr, size, write, false, linear);
728 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
729 struct segmented_address addr,
736 rc = linearize(ctxt, addr, size, false, &linear);
737 if (rc != X86EMUL_CONTINUE)
739 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
743 * Prefetch the remaining bytes of the instruction without crossing page
744 * boundary if they are not in fetch_cache yet.
746 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
750 unsigned long linear;
751 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
752 struct segmented_address addr = { .seg = VCPU_SREG_CS,
753 .ea = ctxt->eip + cur_size };
755 size = 15UL ^ cur_size;
756 rc = __linearize(ctxt, addr, size, false, true, &linear);
757 if (unlikely(rc != X86EMUL_CONTINUE))
760 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
763 * One instruction can only straddle two pages,
764 * and one has been loaded at the beginning of
765 * x86_decode_insn. So, if not enough bytes
766 * still, we must have hit the 15-byte boundary.
768 if (unlikely(size < op_size))
769 return X86EMUL_UNHANDLEABLE;
770 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
771 size, &ctxt->exception);
772 if (unlikely(rc != X86EMUL_CONTINUE))
774 ctxt->fetch.end += size;
775 return X86EMUL_CONTINUE;
778 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
781 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
783 if (unlikely(done_size < size))
784 return __do_insn_fetch_bytes(ctxt, size - done_size);
786 return X86EMUL_CONTINUE;
789 /* Fetch next part of the instruction being emulated. */
790 #define insn_fetch(_type, _ctxt) \
793 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
794 if (rc != X86EMUL_CONTINUE) \
796 ctxt->_eip += sizeof(_type); \
797 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
798 ctxt->fetch.ptr += sizeof(_type); \
802 #define insn_fetch_arr(_arr, _size, _ctxt) \
804 rc = do_insn_fetch_bytes(_ctxt, _size); \
805 if (rc != X86EMUL_CONTINUE) \
807 ctxt->_eip += (_size); \
808 memcpy(_arr, ctxt->fetch.ptr, _size); \
809 ctxt->fetch.ptr += (_size); \
813 * Given the 'reg' portion of a ModRM byte, and a register block, return a
814 * pointer into the block that addresses the relevant register.
815 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
817 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
821 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
823 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
824 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
826 p = reg_rmw(ctxt, modrm_reg);
830 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
831 struct segmented_address addr,
832 u16 *size, unsigned long *address, int op_bytes)
839 rc = segmented_read_std(ctxt, addr, size, 2);
840 if (rc != X86EMUL_CONTINUE)
843 rc = segmented_read_std(ctxt, addr, address, op_bytes);
857 FASTOP1SRC2(mul, mul_ex);
858 FASTOP1SRC2(imul, imul_ex);
859 FASTOP1SRC2EX(div, div_ex);
860 FASTOP1SRC2EX(idiv, idiv_ex);
889 static u8 test_cc(unsigned int condition, unsigned long flags)
892 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
894 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
895 asm("push %[flags]; popf; call *%[fastop]"
896 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
900 static void fetch_register_operand(struct operand *op)
904 op->val = *(u8 *)op->addr.reg;
907 op->val = *(u16 *)op->addr.reg;
910 op->val = *(u32 *)op->addr.reg;
913 op->val = *(u64 *)op->addr.reg;
918 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
920 ctxt->ops->get_fpu(ctxt);
922 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
923 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
924 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
925 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
926 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
927 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
928 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
929 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
931 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
932 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
933 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
934 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
935 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
936 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
937 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
938 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
942 ctxt->ops->put_fpu(ctxt);
945 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
948 ctxt->ops->get_fpu(ctxt);
950 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
951 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
952 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
953 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
954 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
955 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
956 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
957 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
959 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
960 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
961 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
962 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
963 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
964 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
965 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
966 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
970 ctxt->ops->put_fpu(ctxt);
973 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
975 ctxt->ops->get_fpu(ctxt);
977 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
978 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
979 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
980 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
981 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
982 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
983 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
984 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
987 ctxt->ops->put_fpu(ctxt);
990 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
992 ctxt->ops->get_fpu(ctxt);
994 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
995 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
996 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
997 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
998 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
999 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1000 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1001 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1004 ctxt->ops->put_fpu(ctxt);
1007 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1009 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1010 return emulate_nm(ctxt);
1012 ctxt->ops->get_fpu(ctxt);
1013 asm volatile("fninit");
1014 ctxt->ops->put_fpu(ctxt);
1015 return X86EMUL_CONTINUE;
1018 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1022 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1023 return emulate_nm(ctxt);
1025 ctxt->ops->get_fpu(ctxt);
1026 asm volatile("fnstcw %0": "+m"(fcw));
1027 ctxt->ops->put_fpu(ctxt);
1029 /* force 2 byte destination */
1030 ctxt->dst.bytes = 2;
1031 ctxt->dst.val = fcw;
1033 return X86EMUL_CONTINUE;
1036 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1040 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1041 return emulate_nm(ctxt);
1043 ctxt->ops->get_fpu(ctxt);
1044 asm volatile("fnstsw %0": "+m"(fsw));
1045 ctxt->ops->put_fpu(ctxt);
1047 /* force 2 byte destination */
1048 ctxt->dst.bytes = 2;
1049 ctxt->dst.val = fsw;
1051 return X86EMUL_CONTINUE;
1054 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1057 unsigned reg = ctxt->modrm_reg;
1059 if (!(ctxt->d & ModRM))
1060 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1062 if (ctxt->d & Sse) {
1066 read_sse_reg(ctxt, &op->vec_val, reg);
1069 if (ctxt->d & Mmx) {
1078 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1079 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1081 fetch_register_operand(op);
1082 op->orig_val = op->val;
1085 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1087 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1088 ctxt->modrm_seg = VCPU_SREG_SS;
1091 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1095 int index_reg, base_reg, scale;
1096 int rc = X86EMUL_CONTINUE;
1099 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1100 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1101 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1103 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1104 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1105 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1106 ctxt->modrm_seg = VCPU_SREG_DS;
1108 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1110 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1111 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1113 if (ctxt->d & Sse) {
1116 op->addr.xmm = ctxt->modrm_rm;
1117 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1120 if (ctxt->d & Mmx) {
1123 op->addr.mm = ctxt->modrm_rm & 7;
1126 fetch_register_operand(op);
1132 if (ctxt->ad_bytes == 2) {
1133 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1134 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1135 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1136 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1138 /* 16-bit ModR/M decode. */
1139 switch (ctxt->modrm_mod) {
1141 if (ctxt->modrm_rm == 6)
1142 modrm_ea += insn_fetch(u16, ctxt);
1145 modrm_ea += insn_fetch(s8, ctxt);
1148 modrm_ea += insn_fetch(u16, ctxt);
1151 switch (ctxt->modrm_rm) {
1153 modrm_ea += bx + si;
1156 modrm_ea += bx + di;
1159 modrm_ea += bp + si;
1162 modrm_ea += bp + di;
1171 if (ctxt->modrm_mod != 0)
1178 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1179 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1180 ctxt->modrm_seg = VCPU_SREG_SS;
1181 modrm_ea = (u16)modrm_ea;
1183 /* 32/64-bit ModR/M decode. */
1184 if ((ctxt->modrm_rm & 7) == 4) {
1185 sib = insn_fetch(u8, ctxt);
1186 index_reg |= (sib >> 3) & 7;
1187 base_reg |= sib & 7;
1190 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1191 modrm_ea += insn_fetch(s32, ctxt);
1193 modrm_ea += reg_read(ctxt, base_reg);
1194 adjust_modrm_seg(ctxt, base_reg);
1197 modrm_ea += reg_read(ctxt, index_reg) << scale;
1198 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1199 if (ctxt->mode == X86EMUL_MODE_PROT64)
1200 ctxt->rip_relative = 1;
1202 base_reg = ctxt->modrm_rm;
1203 modrm_ea += reg_read(ctxt, base_reg);
1204 adjust_modrm_seg(ctxt, base_reg);
1206 switch (ctxt->modrm_mod) {
1208 if (ctxt->modrm_rm == 5)
1209 modrm_ea += insn_fetch(s32, ctxt);
1212 modrm_ea += insn_fetch(s8, ctxt);
1215 modrm_ea += insn_fetch(s32, ctxt);
1219 op->addr.mem.ea = modrm_ea;
1220 if (ctxt->ad_bytes != 8)
1221 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1227 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1230 int rc = X86EMUL_CONTINUE;
1233 switch (ctxt->ad_bytes) {
1235 op->addr.mem.ea = insn_fetch(u16, ctxt);
1238 op->addr.mem.ea = insn_fetch(u32, ctxt);
1241 op->addr.mem.ea = insn_fetch(u64, ctxt);
1248 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1252 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1253 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1255 if (ctxt->src.bytes == 2)
1256 sv = (s16)ctxt->src.val & (s16)mask;
1257 else if (ctxt->src.bytes == 4)
1258 sv = (s32)ctxt->src.val & (s32)mask;
1260 sv = (s64)ctxt->src.val & (s64)mask;
1262 ctxt->dst.addr.mem.ea += (sv >> 3);
1265 /* only subword offset */
1266 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1269 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1270 unsigned long addr, void *dest, unsigned size)
1273 struct read_cache *mc = &ctxt->mem_read;
1275 if (mc->pos < mc->end)
1278 WARN_ON((mc->end + size) >= sizeof(mc->data));
1280 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1282 if (rc != X86EMUL_CONTINUE)
1288 memcpy(dest, mc->data + mc->pos, size);
1290 return X86EMUL_CONTINUE;
1293 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1294 struct segmented_address addr,
1301 rc = linearize(ctxt, addr, size, false, &linear);
1302 if (rc != X86EMUL_CONTINUE)
1304 return read_emulated(ctxt, linear, data, size);
1307 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1308 struct segmented_address addr,
1315 rc = linearize(ctxt, addr, size, true, &linear);
1316 if (rc != X86EMUL_CONTINUE)
1318 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1322 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1323 struct segmented_address addr,
1324 const void *orig_data, const void *data,
1330 rc = linearize(ctxt, addr, size, true, &linear);
1331 if (rc != X86EMUL_CONTINUE)
1333 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1334 size, &ctxt->exception);
1337 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1338 unsigned int size, unsigned short port,
1341 struct read_cache *rc = &ctxt->io_read;
1343 if (rc->pos == rc->end) { /* refill pio read ahead */
1344 unsigned int in_page, n;
1345 unsigned int count = ctxt->rep_prefix ?
1346 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1347 in_page = (ctxt->eflags & EFLG_DF) ?
1348 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1349 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1350 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1353 rc->pos = rc->end = 0;
1354 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1359 if (ctxt->rep_prefix && (ctxt->d & String) &&
1360 !(ctxt->eflags & EFLG_DF)) {
1361 ctxt->dst.data = rc->data + rc->pos;
1362 ctxt->dst.type = OP_MEM_STR;
1363 ctxt->dst.count = (rc->end - rc->pos) / size;
1366 memcpy(dest, rc->data + rc->pos, size);
1372 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1373 u16 index, struct desc_struct *desc)
1378 ctxt->ops->get_idt(ctxt, &dt);
1380 if (dt.size < index * 8 + 7)
1381 return emulate_gp(ctxt, index << 3 | 0x2);
1383 addr = dt.address + index * 8;
1384 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1388 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1389 u16 selector, struct desc_ptr *dt)
1391 const struct x86_emulate_ops *ops = ctxt->ops;
1394 if (selector & 1 << 2) {
1395 struct desc_struct desc;
1398 memset (dt, 0, sizeof *dt);
1399 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1403 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1404 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1406 ops->get_gdt(ctxt, dt);
1409 /* allowed just for 8 bytes segments */
1410 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1411 u16 selector, struct desc_struct *desc,
1415 u16 index = selector >> 3;
1418 get_descriptor_table_ptr(ctxt, selector, &dt);
1420 if (dt.size < index * 8 + 7)
1421 return emulate_gp(ctxt, selector & 0xfffc);
1423 *desc_addr_p = addr = dt.address + index * 8;
1424 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1428 /* allowed just for 8 bytes segments */
1429 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1430 u16 selector, struct desc_struct *desc)
1433 u16 index = selector >> 3;
1436 get_descriptor_table_ptr(ctxt, selector, &dt);
1438 if (dt.size < index * 8 + 7)
1439 return emulate_gp(ctxt, selector & 0xfffc);
1441 addr = dt.address + index * 8;
1442 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1446 /* Does not support long mode */
1447 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1448 u16 selector, int seg, u8 cpl,
1449 bool in_task_switch,
1450 struct desc_struct *desc)
1452 struct desc_struct seg_desc, old_desc;
1454 unsigned err_vec = GP_VECTOR;
1456 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1462 memset(&seg_desc, 0, sizeof seg_desc);
1464 if (ctxt->mode == X86EMUL_MODE_REAL) {
1465 /* set real mode segment descriptor (keep limit etc. for
1467 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1468 set_desc_base(&seg_desc, selector << 4);
1470 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1471 /* VM86 needs a clean new segment descriptor */
1472 set_desc_base(&seg_desc, selector << 4);
1473 set_desc_limit(&seg_desc, 0xffff);
1483 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1484 if ((seg == VCPU_SREG_CS
1485 || (seg == VCPU_SREG_SS
1486 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1487 || seg == VCPU_SREG_TR)
1491 /* TR should be in GDT only */
1492 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1495 if (null_selector) /* for NULL selector skip all following checks */
1498 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1499 if (ret != X86EMUL_CONTINUE)
1502 err_code = selector & 0xfffc;
1503 err_vec = in_task_switch ? TS_VECTOR : GP_VECTOR;
1505 /* can't load system descriptor into segment selector */
1506 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1510 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1519 * segment is not a writable data segment or segment
1520 * selector's RPL != CPL or segment selector's RPL != CPL
1522 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1526 if (!(seg_desc.type & 8))
1529 if (seg_desc.type & 4) {
1535 if (rpl > cpl || dpl != cpl)
1538 /* in long-mode d/b must be clear if l is set */
1539 if (seg_desc.d && seg_desc.l) {
1542 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1543 if (efer & EFER_LMA)
1547 /* CS(RPL) <- CPL */
1548 selector = (selector & 0xfffc) | cpl;
1551 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1553 old_desc = seg_desc;
1554 seg_desc.type |= 2; /* busy */
1555 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1556 sizeof(seg_desc), &ctxt->exception);
1557 if (ret != X86EMUL_CONTINUE)
1560 case VCPU_SREG_LDTR:
1561 if (seg_desc.s || seg_desc.type != 2)
1564 default: /* DS, ES, FS, or GS */
1566 * segment is not a data or readable code segment or
1567 * ((segment is a data or nonconforming code segment)
1568 * and (both RPL and CPL > DPL))
1570 if ((seg_desc.type & 0xa) == 0x8 ||
1571 (((seg_desc.type & 0xc) != 0xc) &&
1572 (rpl > dpl && cpl > dpl)))
1578 /* mark segment as accessed */
1580 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1581 if (ret != X86EMUL_CONTINUE)
1583 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1584 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1585 sizeof(base3), &ctxt->exception);
1586 if (ret != X86EMUL_CONTINUE)
1590 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1593 return X86EMUL_CONTINUE;
1595 return emulate_exception(ctxt, err_vec, err_code, true);
1598 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1599 u16 selector, int seg)
1601 u8 cpl = ctxt->ops->cpl(ctxt);
1602 return __load_segment_descriptor(ctxt, selector, seg, cpl, false, NULL);
1605 static void write_register_operand(struct operand *op)
1607 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1608 switch (op->bytes) {
1610 *(u8 *)op->addr.reg = (u8)op->val;
1613 *(u16 *)op->addr.reg = (u16)op->val;
1616 *op->addr.reg = (u32)op->val;
1617 break; /* 64b: zero-extend */
1619 *op->addr.reg = op->val;
1624 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1628 write_register_operand(op);
1631 if (ctxt->lock_prefix)
1632 return segmented_cmpxchg(ctxt,
1638 return segmented_write(ctxt,
1644 return segmented_write(ctxt,
1647 op->bytes * op->count);
1650 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1653 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1661 return X86EMUL_CONTINUE;
1664 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1666 struct segmented_address addr;
1668 rsp_increment(ctxt, -bytes);
1669 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1670 addr.seg = VCPU_SREG_SS;
1672 return segmented_write(ctxt, addr, data, bytes);
1675 static int em_push(struct x86_emulate_ctxt *ctxt)
1677 /* Disable writeback. */
1678 ctxt->dst.type = OP_NONE;
1679 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1682 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1683 void *dest, int len)
1686 struct segmented_address addr;
1688 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1689 addr.seg = VCPU_SREG_SS;
1690 rc = segmented_read(ctxt, addr, dest, len);
1691 if (rc != X86EMUL_CONTINUE)
1694 rsp_increment(ctxt, len);
1698 static int em_pop(struct x86_emulate_ctxt *ctxt)
1700 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1703 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1704 void *dest, int len)
1707 unsigned long val, change_mask;
1708 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1709 int cpl = ctxt->ops->cpl(ctxt);
1711 rc = emulate_pop(ctxt, &val, len);
1712 if (rc != X86EMUL_CONTINUE)
1715 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1716 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1718 switch(ctxt->mode) {
1719 case X86EMUL_MODE_PROT64:
1720 case X86EMUL_MODE_PROT32:
1721 case X86EMUL_MODE_PROT16:
1723 change_mask |= EFLG_IOPL;
1725 change_mask |= EFLG_IF;
1727 case X86EMUL_MODE_VM86:
1729 return emulate_gp(ctxt, 0);
1730 change_mask |= EFLG_IF;
1732 default: /* real mode */
1733 change_mask |= (EFLG_IOPL | EFLG_IF);
1737 *(unsigned long *)dest =
1738 (ctxt->eflags & ~change_mask) | (val & change_mask);
1743 static int em_popf(struct x86_emulate_ctxt *ctxt)
1745 ctxt->dst.type = OP_REG;
1746 ctxt->dst.addr.reg = &ctxt->eflags;
1747 ctxt->dst.bytes = ctxt->op_bytes;
1748 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1751 static int em_enter(struct x86_emulate_ctxt *ctxt)
1754 unsigned frame_size = ctxt->src.val;
1755 unsigned nesting_level = ctxt->src2.val & 31;
1759 return X86EMUL_UNHANDLEABLE;
1761 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1762 rc = push(ctxt, &rbp, stack_size(ctxt));
1763 if (rc != X86EMUL_CONTINUE)
1765 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1767 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1768 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1770 return X86EMUL_CONTINUE;
1773 static int em_leave(struct x86_emulate_ctxt *ctxt)
1775 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1777 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1780 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1782 int seg = ctxt->src2.val;
1784 ctxt->src.val = get_segment_selector(ctxt, seg);
1786 return em_push(ctxt);
1789 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1791 int seg = ctxt->src2.val;
1792 unsigned long selector;
1795 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1796 if (rc != X86EMUL_CONTINUE)
1799 if (ctxt->modrm_reg == VCPU_SREG_SS)
1800 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1802 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1806 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1808 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1809 int rc = X86EMUL_CONTINUE;
1810 int reg = VCPU_REGS_RAX;
1812 while (reg <= VCPU_REGS_RDI) {
1813 (reg == VCPU_REGS_RSP) ?
1814 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1817 if (rc != X86EMUL_CONTINUE)
1826 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1828 ctxt->src.val = (unsigned long)ctxt->eflags;
1829 return em_push(ctxt);
1832 static int em_popa(struct x86_emulate_ctxt *ctxt)
1834 int rc = X86EMUL_CONTINUE;
1835 int reg = VCPU_REGS_RDI;
1837 while (reg >= VCPU_REGS_RAX) {
1838 if (reg == VCPU_REGS_RSP) {
1839 rsp_increment(ctxt, ctxt->op_bytes);
1843 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1844 if (rc != X86EMUL_CONTINUE)
1851 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1853 const struct x86_emulate_ops *ops = ctxt->ops;
1860 /* TODO: Add limit checks */
1861 ctxt->src.val = ctxt->eflags;
1863 if (rc != X86EMUL_CONTINUE)
1866 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1868 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1870 if (rc != X86EMUL_CONTINUE)
1873 ctxt->src.val = ctxt->_eip;
1875 if (rc != X86EMUL_CONTINUE)
1878 ops->get_idt(ctxt, &dt);
1880 eip_addr = dt.address + (irq << 2);
1881 cs_addr = dt.address + (irq << 2) + 2;
1883 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1884 if (rc != X86EMUL_CONTINUE)
1887 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1888 if (rc != X86EMUL_CONTINUE)
1891 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1892 if (rc != X86EMUL_CONTINUE)
1900 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1904 invalidate_registers(ctxt);
1905 rc = __emulate_int_real(ctxt, irq);
1906 if (rc == X86EMUL_CONTINUE)
1907 writeback_registers(ctxt);
1911 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1913 switch(ctxt->mode) {
1914 case X86EMUL_MODE_REAL:
1915 return __emulate_int_real(ctxt, irq);
1916 case X86EMUL_MODE_VM86:
1917 case X86EMUL_MODE_PROT16:
1918 case X86EMUL_MODE_PROT32:
1919 case X86EMUL_MODE_PROT64:
1921 /* Protected mode interrupts unimplemented yet */
1922 return X86EMUL_UNHANDLEABLE;
1926 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1928 int rc = X86EMUL_CONTINUE;
1929 unsigned long temp_eip = 0;
1930 unsigned long temp_eflags = 0;
1931 unsigned long cs = 0;
1932 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1933 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1934 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1935 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1937 /* TODO: Add stack limit check */
1939 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1941 if (rc != X86EMUL_CONTINUE)
1944 if (temp_eip & ~0xffff)
1945 return emulate_gp(ctxt, 0);
1947 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1949 if (rc != X86EMUL_CONTINUE)
1952 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1954 if (rc != X86EMUL_CONTINUE)
1957 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1959 if (rc != X86EMUL_CONTINUE)
1962 ctxt->_eip = temp_eip;
1965 if (ctxt->op_bytes == 4)
1966 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1967 else if (ctxt->op_bytes == 2) {
1968 ctxt->eflags &= ~0xffff;
1969 ctxt->eflags |= temp_eflags;
1972 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1973 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1978 static int em_iret(struct x86_emulate_ctxt *ctxt)
1980 switch(ctxt->mode) {
1981 case X86EMUL_MODE_REAL:
1982 return emulate_iret_real(ctxt);
1983 case X86EMUL_MODE_VM86:
1984 case X86EMUL_MODE_PROT16:
1985 case X86EMUL_MODE_PROT32:
1986 case X86EMUL_MODE_PROT64:
1988 /* iret from protected mode unimplemented yet */
1989 return X86EMUL_UNHANDLEABLE;
1993 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1996 unsigned short sel, old_sel;
1997 struct desc_struct old_desc, new_desc;
1998 const struct x86_emulate_ops *ops = ctxt->ops;
1999 u8 cpl = ctxt->ops->cpl(ctxt);
2001 /* Assignment of RIP may only fail in 64-bit mode */
2002 if (ctxt->mode == X86EMUL_MODE_PROT64)
2003 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2006 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2008 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, false,
2010 if (rc != X86EMUL_CONTINUE)
2013 rc = assign_eip_far(ctxt, ctxt->src.val, new_desc.l);
2014 if (rc != X86EMUL_CONTINUE) {
2015 WARN_ON(!ctxt->mode != X86EMUL_MODE_PROT64);
2016 /* assigning eip failed; restore the old cs */
2017 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2023 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2025 int rc = X86EMUL_CONTINUE;
2027 switch (ctxt->modrm_reg) {
2028 case 2: /* call near abs */ {
2030 old_eip = ctxt->_eip;
2031 rc = assign_eip_near(ctxt, ctxt->src.val);
2032 if (rc != X86EMUL_CONTINUE)
2034 ctxt->src.val = old_eip;
2038 case 4: /* jmp abs */
2039 rc = assign_eip_near(ctxt, ctxt->src.val);
2041 case 5: /* jmp far */
2042 rc = em_jmp_far(ctxt);
2051 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2053 u64 old = ctxt->dst.orig_val64;
2055 if (ctxt->dst.bytes == 16)
2056 return X86EMUL_UNHANDLEABLE;
2058 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2059 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2060 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2061 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2062 ctxt->eflags &= ~EFLG_ZF;
2064 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2065 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2067 ctxt->eflags |= EFLG_ZF;
2069 return X86EMUL_CONTINUE;
2072 static int em_ret(struct x86_emulate_ctxt *ctxt)
2077 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2078 if (rc != X86EMUL_CONTINUE)
2081 return assign_eip_near(ctxt, eip);
2084 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2087 unsigned long eip, cs;
2089 int cpl = ctxt->ops->cpl(ctxt);
2090 struct desc_struct old_desc, new_desc;
2091 const struct x86_emulate_ops *ops = ctxt->ops;
2093 if (ctxt->mode == X86EMUL_MODE_PROT64)
2094 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2097 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2098 if (rc != X86EMUL_CONTINUE)
2100 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2101 if (rc != X86EMUL_CONTINUE)
2103 /* Outer-privilege level return is not implemented */
2104 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2105 return X86EMUL_UNHANDLEABLE;
2106 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, 0, false,
2108 if (rc != X86EMUL_CONTINUE)
2110 rc = assign_eip_far(ctxt, eip, new_desc.l);
2111 if (rc != X86EMUL_CONTINUE) {
2112 WARN_ON(!ctxt->mode != X86EMUL_MODE_PROT64);
2113 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2118 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2122 rc = em_ret_far(ctxt);
2123 if (rc != X86EMUL_CONTINUE)
2125 rsp_increment(ctxt, ctxt->src.val);
2126 return X86EMUL_CONTINUE;
2129 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2131 /* Save real source value, then compare EAX against destination. */
2132 ctxt->dst.orig_val = ctxt->dst.val;
2133 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2134 ctxt->src.orig_val = ctxt->src.val;
2135 ctxt->src.val = ctxt->dst.orig_val;
2136 fastop(ctxt, em_cmp);
2138 if (ctxt->eflags & EFLG_ZF) {
2139 /* Success: write back to memory. */
2140 ctxt->dst.val = ctxt->src.orig_val;
2142 /* Failure: write the value we saw to EAX. */
2143 ctxt->dst.type = OP_REG;
2144 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2145 ctxt->dst.val = ctxt->dst.orig_val;
2147 return X86EMUL_CONTINUE;
2150 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2152 int seg = ctxt->src2.val;
2156 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2158 rc = load_segment_descriptor(ctxt, sel, seg);
2159 if (rc != X86EMUL_CONTINUE)
2162 ctxt->dst.val = ctxt->src.val;
2167 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2168 struct desc_struct *cs, struct desc_struct *ss)
2170 cs->l = 0; /* will be adjusted later */
2171 set_desc_base(cs, 0); /* flat segment */
2172 cs->g = 1; /* 4kb granularity */
2173 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2174 cs->type = 0x0b; /* Read, Execute, Accessed */
2176 cs->dpl = 0; /* will be adjusted later */
2181 set_desc_base(ss, 0); /* flat segment */
2182 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2183 ss->g = 1; /* 4kb granularity */
2185 ss->type = 0x03; /* Read/Write, Accessed */
2186 ss->d = 1; /* 32bit stack segment */
2193 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2195 u32 eax, ebx, ecx, edx;
2198 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2199 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2200 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2201 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2204 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2206 const struct x86_emulate_ops *ops = ctxt->ops;
2207 u32 eax, ebx, ecx, edx;
2210 * syscall should always be enabled in longmode - so only become
2211 * vendor specific (cpuid) if other modes are active...
2213 if (ctxt->mode == X86EMUL_MODE_PROT64)
2218 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2220 * Intel ("GenuineIntel")
2221 * remark: Intel CPUs only support "syscall" in 64bit
2222 * longmode. Also an 64bit guest with a
2223 * 32bit compat-app running will #UD !! While this
2224 * behaviour can be fixed (by emulating) into AMD
2225 * response - CPUs of AMD can't behave like Intel.
2227 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2228 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2229 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2232 /* AMD ("AuthenticAMD") */
2233 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2234 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2235 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2238 /* AMD ("AMDisbetter!") */
2239 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2240 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2241 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2244 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2248 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2250 const struct x86_emulate_ops *ops = ctxt->ops;
2251 struct desc_struct cs, ss;
2256 /* syscall is not available in real mode */
2257 if (ctxt->mode == X86EMUL_MODE_REAL ||
2258 ctxt->mode == X86EMUL_MODE_VM86)
2259 return emulate_ud(ctxt);
2261 if (!(em_syscall_is_enabled(ctxt)))
2262 return emulate_ud(ctxt);
2264 ops->get_msr(ctxt, MSR_EFER, &efer);
2265 setup_syscalls_segments(ctxt, &cs, &ss);
2267 if (!(efer & EFER_SCE))
2268 return emulate_ud(ctxt);
2270 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2272 cs_sel = (u16)(msr_data & 0xfffc);
2273 ss_sel = (u16)(msr_data + 8);
2275 if (efer & EFER_LMA) {
2279 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2280 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2282 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2283 if (efer & EFER_LMA) {
2284 #ifdef CONFIG_X86_64
2285 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2288 ctxt->mode == X86EMUL_MODE_PROT64 ?
2289 MSR_LSTAR : MSR_CSTAR, &msr_data);
2290 ctxt->_eip = msr_data;
2292 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2293 ctxt->eflags &= ~msr_data;
2297 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2298 ctxt->_eip = (u32)msr_data;
2300 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2303 return X86EMUL_CONTINUE;
2306 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2308 const struct x86_emulate_ops *ops = ctxt->ops;
2309 struct desc_struct cs, ss;
2314 ops->get_msr(ctxt, MSR_EFER, &efer);
2315 /* inject #GP if in real mode */
2316 if (ctxt->mode == X86EMUL_MODE_REAL)
2317 return emulate_gp(ctxt, 0);
2320 * Not recognized on AMD in compat mode (but is recognized in legacy
2323 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2324 && !vendor_intel(ctxt))
2325 return emulate_ud(ctxt);
2327 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2328 * Therefore, we inject an #UD.
2330 if (ctxt->mode == X86EMUL_MODE_PROT64)
2331 return emulate_ud(ctxt);
2333 setup_syscalls_segments(ctxt, &cs, &ss);
2335 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2336 switch (ctxt->mode) {
2337 case X86EMUL_MODE_PROT32:
2338 if ((msr_data & 0xfffc) == 0x0)
2339 return emulate_gp(ctxt, 0);
2341 case X86EMUL_MODE_PROT64:
2342 if (msr_data == 0x0)
2343 return emulate_gp(ctxt, 0);
2349 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2350 cs_sel = (u16)msr_data;
2351 cs_sel &= ~SELECTOR_RPL_MASK;
2352 ss_sel = cs_sel + 8;
2353 ss_sel &= ~SELECTOR_RPL_MASK;
2354 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2359 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2360 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2362 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2363 ctxt->_eip = msr_data;
2365 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2366 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2368 return X86EMUL_CONTINUE;
2371 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2373 const struct x86_emulate_ops *ops = ctxt->ops;
2374 struct desc_struct cs, ss;
2375 u64 msr_data, rcx, rdx;
2377 u16 cs_sel = 0, ss_sel = 0;
2379 /* inject #GP if in real mode or Virtual 8086 mode */
2380 if (ctxt->mode == X86EMUL_MODE_REAL ||
2381 ctxt->mode == X86EMUL_MODE_VM86)
2382 return emulate_gp(ctxt, 0);
2384 setup_syscalls_segments(ctxt, &cs, &ss);
2386 if ((ctxt->rex_prefix & 0x8) != 0x0)
2387 usermode = X86EMUL_MODE_PROT64;
2389 usermode = X86EMUL_MODE_PROT32;
2391 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2392 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2396 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2398 case X86EMUL_MODE_PROT32:
2399 cs_sel = (u16)(msr_data + 16);
2400 if ((msr_data & 0xfffc) == 0x0)
2401 return emulate_gp(ctxt, 0);
2402 ss_sel = (u16)(msr_data + 24);
2404 case X86EMUL_MODE_PROT64:
2405 cs_sel = (u16)(msr_data + 32);
2406 if (msr_data == 0x0)
2407 return emulate_gp(ctxt, 0);
2408 ss_sel = cs_sel + 8;
2411 if (is_noncanonical_address(rcx) ||
2412 is_noncanonical_address(rdx))
2413 return emulate_gp(ctxt, 0);
2416 cs_sel |= SELECTOR_RPL_MASK;
2417 ss_sel |= SELECTOR_RPL_MASK;
2419 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2420 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2423 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2425 return X86EMUL_CONTINUE;
2428 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2431 if (ctxt->mode == X86EMUL_MODE_REAL)
2433 if (ctxt->mode == X86EMUL_MODE_VM86)
2435 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2436 return ctxt->ops->cpl(ctxt) > iopl;
2439 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2442 const struct x86_emulate_ops *ops = ctxt->ops;
2443 struct desc_struct tr_seg;
2446 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2447 unsigned mask = (1 << len) - 1;
2450 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2453 if (desc_limit_scaled(&tr_seg) < 103)
2455 base = get_desc_base(&tr_seg);
2456 #ifdef CONFIG_X86_64
2457 base |= ((u64)base3) << 32;
2459 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2460 if (r != X86EMUL_CONTINUE)
2462 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2464 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2465 if (r != X86EMUL_CONTINUE)
2467 if ((perm >> bit_idx) & mask)
2472 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2478 if (emulator_bad_iopl(ctxt))
2479 if (!emulator_io_port_access_allowed(ctxt, port, len))
2482 ctxt->perm_ok = true;
2487 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2488 struct tss_segment_16 *tss)
2490 tss->ip = ctxt->_eip;
2491 tss->flag = ctxt->eflags;
2492 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2493 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2494 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2495 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2496 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2497 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2498 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2499 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2501 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2502 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2503 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2504 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2505 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2508 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2509 struct tss_segment_16 *tss)
2514 ctxt->_eip = tss->ip;
2515 ctxt->eflags = tss->flag | 2;
2516 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2517 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2518 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2519 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2520 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2521 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2522 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2523 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2526 * SDM says that segment selectors are loaded before segment
2529 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2530 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2531 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2532 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2533 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2538 * Now load segment descriptors. If fault happens at this stage
2539 * it is handled in a context of new task
2541 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2543 if (ret != X86EMUL_CONTINUE)
2545 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2547 if (ret != X86EMUL_CONTINUE)
2549 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2551 if (ret != X86EMUL_CONTINUE)
2553 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2555 if (ret != X86EMUL_CONTINUE)
2557 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2559 if (ret != X86EMUL_CONTINUE)
2562 return X86EMUL_CONTINUE;
2565 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2566 u16 tss_selector, u16 old_tss_sel,
2567 ulong old_tss_base, struct desc_struct *new_desc)
2569 const struct x86_emulate_ops *ops = ctxt->ops;
2570 struct tss_segment_16 tss_seg;
2572 u32 new_tss_base = get_desc_base(new_desc);
2574 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2576 if (ret != X86EMUL_CONTINUE)
2577 /* FIXME: need to provide precise fault address */
2580 save_state_to_tss16(ctxt, &tss_seg);
2582 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2584 if (ret != X86EMUL_CONTINUE)
2585 /* FIXME: need to provide precise fault address */
2588 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2590 if (ret != X86EMUL_CONTINUE)
2591 /* FIXME: need to provide precise fault address */
2594 if (old_tss_sel != 0xffff) {
2595 tss_seg.prev_task_link = old_tss_sel;
2597 ret = ops->write_std(ctxt, new_tss_base,
2598 &tss_seg.prev_task_link,
2599 sizeof tss_seg.prev_task_link,
2601 if (ret != X86EMUL_CONTINUE)
2602 /* FIXME: need to provide precise fault address */
2606 return load_state_from_tss16(ctxt, &tss_seg);
2609 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2610 struct tss_segment_32 *tss)
2612 /* CR3 and ldt selector are not saved intentionally */
2613 tss->eip = ctxt->_eip;
2614 tss->eflags = ctxt->eflags;
2615 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2616 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2617 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2618 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2619 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2620 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2621 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2622 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2624 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2625 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2626 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2627 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2628 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2629 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2632 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2633 struct tss_segment_32 *tss)
2638 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2639 return emulate_gp(ctxt, 0);
2640 ctxt->_eip = tss->eip;
2641 ctxt->eflags = tss->eflags | 2;
2643 /* General purpose registers */
2644 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2645 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2646 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2647 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2648 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2649 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2650 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2651 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2654 * SDM says that segment selectors are loaded before segment
2655 * descriptors. This is important because CPL checks will
2658 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2659 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2660 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2661 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2662 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2663 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2664 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2667 * If we're switching between Protected Mode and VM86, we need to make
2668 * sure to update the mode before loading the segment descriptors so
2669 * that the selectors are interpreted correctly.
2671 if (ctxt->eflags & X86_EFLAGS_VM) {
2672 ctxt->mode = X86EMUL_MODE_VM86;
2675 ctxt->mode = X86EMUL_MODE_PROT32;
2680 * Now load segment descriptors. If fault happenes at this stage
2681 * it is handled in a context of new task
2683 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2685 if (ret != X86EMUL_CONTINUE)
2687 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2689 if (ret != X86EMUL_CONTINUE)
2691 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2693 if (ret != X86EMUL_CONTINUE)
2695 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2697 if (ret != X86EMUL_CONTINUE)
2699 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2701 if (ret != X86EMUL_CONTINUE)
2703 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2705 if (ret != X86EMUL_CONTINUE)
2707 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2709 if (ret != X86EMUL_CONTINUE)
2712 return X86EMUL_CONTINUE;
2715 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2716 u16 tss_selector, u16 old_tss_sel,
2717 ulong old_tss_base, struct desc_struct *new_desc)
2719 const struct x86_emulate_ops *ops = ctxt->ops;
2720 struct tss_segment_32 tss_seg;
2722 u32 new_tss_base = get_desc_base(new_desc);
2723 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2724 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2726 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2728 if (ret != X86EMUL_CONTINUE)
2729 /* FIXME: need to provide precise fault address */
2732 save_state_to_tss32(ctxt, &tss_seg);
2734 /* Only GP registers and segment selectors are saved */
2735 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2736 ldt_sel_offset - eip_offset, &ctxt->exception);
2737 if (ret != X86EMUL_CONTINUE)
2738 /* FIXME: need to provide precise fault address */
2741 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2743 if (ret != X86EMUL_CONTINUE)
2744 /* FIXME: need to provide precise fault address */
2747 if (old_tss_sel != 0xffff) {
2748 tss_seg.prev_task_link = old_tss_sel;
2750 ret = ops->write_std(ctxt, new_tss_base,
2751 &tss_seg.prev_task_link,
2752 sizeof tss_seg.prev_task_link,
2754 if (ret != X86EMUL_CONTINUE)
2755 /* FIXME: need to provide precise fault address */
2759 return load_state_from_tss32(ctxt, &tss_seg);
2762 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2763 u16 tss_selector, int idt_index, int reason,
2764 bool has_error_code, u32 error_code)
2766 const struct x86_emulate_ops *ops = ctxt->ops;
2767 struct desc_struct curr_tss_desc, next_tss_desc;
2769 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2770 ulong old_tss_base =
2771 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2775 /* FIXME: old_tss_base == ~0 ? */
2777 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2778 if (ret != X86EMUL_CONTINUE)
2780 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2781 if (ret != X86EMUL_CONTINUE)
2784 /* FIXME: check that next_tss_desc is tss */
2787 * Check privileges. The three cases are task switch caused by...
2789 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2790 * 2. Exception/IRQ/iret: No check is performed
2791 * 3. jmp/call to TSS: Check against DPL of the TSS
2793 if (reason == TASK_SWITCH_GATE) {
2794 if (idt_index != -1) {
2795 /* Software interrupts */
2796 struct desc_struct task_gate_desc;
2799 ret = read_interrupt_descriptor(ctxt, idt_index,
2801 if (ret != X86EMUL_CONTINUE)
2804 dpl = task_gate_desc.dpl;
2805 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2806 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2808 } else if (reason != TASK_SWITCH_IRET) {
2809 int dpl = next_tss_desc.dpl;
2810 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2811 return emulate_gp(ctxt, tss_selector);
2815 desc_limit = desc_limit_scaled(&next_tss_desc);
2816 if (!next_tss_desc.p ||
2817 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2818 desc_limit < 0x2b)) {
2819 return emulate_ts(ctxt, tss_selector & 0xfffc);
2822 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2823 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2824 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2827 if (reason == TASK_SWITCH_IRET)
2828 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2830 /* set back link to prev task only if NT bit is set in eflags
2831 note that old_tss_sel is not used after this point */
2832 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2833 old_tss_sel = 0xffff;
2835 if (next_tss_desc.type & 8)
2836 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2837 old_tss_base, &next_tss_desc);
2839 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2840 old_tss_base, &next_tss_desc);
2841 if (ret != X86EMUL_CONTINUE)
2844 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2845 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2847 if (reason != TASK_SWITCH_IRET) {
2848 next_tss_desc.type |= (1 << 1); /* set busy flag */
2849 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2852 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2853 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2855 if (has_error_code) {
2856 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2857 ctxt->lock_prefix = 0;
2858 ctxt->src.val = (unsigned long) error_code;
2859 ret = em_push(ctxt);
2865 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2866 u16 tss_selector, int idt_index, int reason,
2867 bool has_error_code, u32 error_code)
2871 invalidate_registers(ctxt);
2872 ctxt->_eip = ctxt->eip;
2873 ctxt->dst.type = OP_NONE;
2875 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2876 has_error_code, error_code);
2878 if (rc == X86EMUL_CONTINUE) {
2879 ctxt->eip = ctxt->_eip;
2880 writeback_registers(ctxt);
2883 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2886 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2889 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2891 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2892 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2895 static int em_das(struct x86_emulate_ctxt *ctxt)
2898 bool af, cf, old_cf;
2900 cf = ctxt->eflags & X86_EFLAGS_CF;
2906 af = ctxt->eflags & X86_EFLAGS_AF;
2907 if ((al & 0x0f) > 9 || af) {
2909 cf = old_cf | (al >= 250);
2914 if (old_al > 0x99 || old_cf) {
2920 /* Set PF, ZF, SF */
2921 ctxt->src.type = OP_IMM;
2923 ctxt->src.bytes = 1;
2924 fastop(ctxt, em_or);
2925 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2927 ctxt->eflags |= X86_EFLAGS_CF;
2929 ctxt->eflags |= X86_EFLAGS_AF;
2930 return X86EMUL_CONTINUE;
2933 static int em_aam(struct x86_emulate_ctxt *ctxt)
2937 if (ctxt->src.val == 0)
2938 return emulate_de(ctxt);
2940 al = ctxt->dst.val & 0xff;
2941 ah = al / ctxt->src.val;
2942 al %= ctxt->src.val;
2944 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2946 /* Set PF, ZF, SF */
2947 ctxt->src.type = OP_IMM;
2949 ctxt->src.bytes = 1;
2950 fastop(ctxt, em_or);
2952 return X86EMUL_CONTINUE;
2955 static int em_aad(struct x86_emulate_ctxt *ctxt)
2957 u8 al = ctxt->dst.val & 0xff;
2958 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2960 al = (al + (ah * ctxt->src.val)) & 0xff;
2962 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2964 /* Set PF, ZF, SF */
2965 ctxt->src.type = OP_IMM;
2967 ctxt->src.bytes = 1;
2968 fastop(ctxt, em_or);
2970 return X86EMUL_CONTINUE;
2973 static int em_call(struct x86_emulate_ctxt *ctxt)
2976 long rel = ctxt->src.val;
2978 ctxt->src.val = (unsigned long)ctxt->_eip;
2979 rc = jmp_rel(ctxt, rel);
2980 if (rc != X86EMUL_CONTINUE)
2982 return em_push(ctxt);
2985 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2990 struct desc_struct old_desc, new_desc;
2991 const struct x86_emulate_ops *ops = ctxt->ops;
2992 int cpl = ctxt->ops->cpl(ctxt);
2994 old_eip = ctxt->_eip;
2995 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
2997 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2998 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, false,
3000 if (rc != X86EMUL_CONTINUE)
3001 return X86EMUL_CONTINUE;
3003 rc = assign_eip_far(ctxt, ctxt->src.val, new_desc.l);
3004 if (rc != X86EMUL_CONTINUE)
3007 ctxt->src.val = old_cs;
3009 if (rc != X86EMUL_CONTINUE)
3012 ctxt->src.val = old_eip;
3014 /* If we failed, we tainted the memory, but the very least we should
3016 if (rc != X86EMUL_CONTINUE)
3020 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3025 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3030 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3031 if (rc != X86EMUL_CONTINUE)
3033 rc = assign_eip_near(ctxt, eip);
3034 if (rc != X86EMUL_CONTINUE)
3036 rsp_increment(ctxt, ctxt->src.val);
3037 return X86EMUL_CONTINUE;
3040 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3042 /* Write back the register source. */
3043 ctxt->src.val = ctxt->dst.val;
3044 write_register_operand(&ctxt->src);
3046 /* Write back the memory destination with implicit LOCK prefix. */
3047 ctxt->dst.val = ctxt->src.orig_val;
3048 ctxt->lock_prefix = 1;
3049 return X86EMUL_CONTINUE;
3052 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3054 ctxt->dst.val = ctxt->src2.val;
3055 return fastop(ctxt, em_imul);
3058 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3060 ctxt->dst.type = OP_REG;
3061 ctxt->dst.bytes = ctxt->src.bytes;
3062 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3063 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3065 return X86EMUL_CONTINUE;
3068 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3072 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3073 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3074 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3075 return X86EMUL_CONTINUE;
3078 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3082 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3083 return emulate_gp(ctxt, 0);
3084 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3085 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3086 return X86EMUL_CONTINUE;
3089 static int em_mov(struct x86_emulate_ctxt *ctxt)
3091 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3092 return X86EMUL_CONTINUE;
3095 #define FFL(x) bit(X86_FEATURE_##x)
3097 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3099 u32 ebx, ecx, edx, eax = 1;
3103 * Check MOVBE is set in the guest-visible CPUID leaf.
3105 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3106 if (!(ecx & FFL(MOVBE)))
3107 return emulate_ud(ctxt);
3109 switch (ctxt->op_bytes) {
3112 * From MOVBE definition: "...When the operand size is 16 bits,
3113 * the upper word of the destination register remains unchanged
3116 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3117 * rules so we have to do the operation almost per hand.
3119 tmp = (u16)ctxt->src.val;
3120 ctxt->dst.val &= ~0xffffUL;
3121 ctxt->dst.val |= (unsigned long)swab16(tmp);
3124 ctxt->dst.val = swab32((u32)ctxt->src.val);
3127 ctxt->dst.val = swab64(ctxt->src.val);
3132 return X86EMUL_CONTINUE;
3135 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3137 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3138 return emulate_gp(ctxt, 0);
3140 /* Disable writeback. */
3141 ctxt->dst.type = OP_NONE;
3142 return X86EMUL_CONTINUE;
3145 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3149 if (ctxt->mode == X86EMUL_MODE_PROT64)
3150 val = ctxt->src.val & ~0ULL;
3152 val = ctxt->src.val & ~0U;
3154 /* #UD condition is already handled. */
3155 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3156 return emulate_gp(ctxt, 0);
3158 /* Disable writeback. */
3159 ctxt->dst.type = OP_NONE;
3160 return X86EMUL_CONTINUE;
3163 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3167 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3168 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3169 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3170 return emulate_gp(ctxt, 0);
3172 return X86EMUL_CONTINUE;
3175 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3179 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3180 return emulate_gp(ctxt, 0);
3182 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3183 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3184 return X86EMUL_CONTINUE;
3187 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3189 if (ctxt->modrm_reg > VCPU_SREG_GS)
3190 return emulate_ud(ctxt);
3192 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3193 return X86EMUL_CONTINUE;
3196 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3198 u16 sel = ctxt->src.val;
3200 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3201 return emulate_ud(ctxt);
3203 if (ctxt->modrm_reg == VCPU_SREG_SS)
3204 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3206 /* Disable writeback. */
3207 ctxt->dst.type = OP_NONE;
3208 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3211 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3213 u16 sel = ctxt->src.val;
3215 /* Disable writeback. */
3216 ctxt->dst.type = OP_NONE;
3217 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3220 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3222 u16 sel = ctxt->src.val;
3224 /* Disable writeback. */
3225 ctxt->dst.type = OP_NONE;
3226 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3229 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3234 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3235 if (rc == X86EMUL_CONTINUE)
3236 ctxt->ops->invlpg(ctxt, linear);
3237 /* Disable writeback. */
3238 ctxt->dst.type = OP_NONE;
3239 return X86EMUL_CONTINUE;
3242 static int em_clts(struct x86_emulate_ctxt *ctxt)
3246 cr0 = ctxt->ops->get_cr(ctxt, 0);
3248 ctxt->ops->set_cr(ctxt, 0, cr0);
3249 return X86EMUL_CONTINUE;
3252 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3254 int rc = ctxt->ops->fix_hypercall(ctxt);
3256 if (rc != X86EMUL_CONTINUE)
3259 /* Let the processor re-execute the fixed hypercall */
3260 ctxt->_eip = ctxt->eip;
3261 /* Disable writeback. */
3262 ctxt->dst.type = OP_NONE;
3263 return X86EMUL_CONTINUE;
3266 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3267 void (*get)(struct x86_emulate_ctxt *ctxt,
3268 struct desc_ptr *ptr))
3270 struct desc_ptr desc_ptr;
3272 if (ctxt->mode == X86EMUL_MODE_PROT64)
3274 get(ctxt, &desc_ptr);
3275 if (ctxt->op_bytes == 2) {
3277 desc_ptr.address &= 0x00ffffff;
3279 /* Disable writeback. */
3280 ctxt->dst.type = OP_NONE;
3281 return segmented_write(ctxt, ctxt->dst.addr.mem,
3282 &desc_ptr, 2 + ctxt->op_bytes);
3285 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3287 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3290 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3292 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3295 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3297 struct desc_ptr desc_ptr;
3300 if (ctxt->mode == X86EMUL_MODE_PROT64)
3302 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3303 &desc_ptr.size, &desc_ptr.address,
3305 if (rc != X86EMUL_CONTINUE)
3307 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3308 /* Disable writeback. */
3309 ctxt->dst.type = OP_NONE;
3310 return X86EMUL_CONTINUE;
3313 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3317 rc = ctxt->ops->fix_hypercall(ctxt);
3319 /* Disable writeback. */
3320 ctxt->dst.type = OP_NONE;
3324 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3326 struct desc_ptr desc_ptr;
3329 if (ctxt->mode == X86EMUL_MODE_PROT64)
3331 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3332 &desc_ptr.size, &desc_ptr.address,
3334 if (rc != X86EMUL_CONTINUE)
3336 ctxt->ops->set_idt(ctxt, &desc_ptr);
3337 /* Disable writeback. */
3338 ctxt->dst.type = OP_NONE;
3339 return X86EMUL_CONTINUE;
3342 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3344 if (ctxt->dst.type == OP_MEM)
3345 ctxt->dst.bytes = 2;
3346 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3347 return X86EMUL_CONTINUE;
3350 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3352 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3353 | (ctxt->src.val & 0x0f));
3354 ctxt->dst.type = OP_NONE;
3355 return X86EMUL_CONTINUE;
3358 static int em_loop(struct x86_emulate_ctxt *ctxt)
3360 int rc = X86EMUL_CONTINUE;
3362 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3363 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3364 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3365 rc = jmp_rel(ctxt, ctxt->src.val);
3370 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3372 int rc = X86EMUL_CONTINUE;
3374 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3375 rc = jmp_rel(ctxt, ctxt->src.val);
3380 static int em_in(struct x86_emulate_ctxt *ctxt)
3382 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3384 return X86EMUL_IO_NEEDED;
3386 return X86EMUL_CONTINUE;
3389 static int em_out(struct x86_emulate_ctxt *ctxt)
3391 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3393 /* Disable writeback. */
3394 ctxt->dst.type = OP_NONE;
3395 return X86EMUL_CONTINUE;
3398 static int em_cli(struct x86_emulate_ctxt *ctxt)
3400 if (emulator_bad_iopl(ctxt))
3401 return emulate_gp(ctxt, 0);
3403 ctxt->eflags &= ~X86_EFLAGS_IF;
3404 return X86EMUL_CONTINUE;
3407 static int em_sti(struct x86_emulate_ctxt *ctxt)
3409 if (emulator_bad_iopl(ctxt))
3410 return emulate_gp(ctxt, 0);
3412 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3413 ctxt->eflags |= X86_EFLAGS_IF;
3414 return X86EMUL_CONTINUE;
3417 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3419 u32 eax, ebx, ecx, edx;
3421 eax = reg_read(ctxt, VCPU_REGS_RAX);
3422 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3423 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3424 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3425 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3426 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3427 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3428 return X86EMUL_CONTINUE;
3431 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3435 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3436 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3438 ctxt->eflags &= ~0xffUL;
3439 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3440 return X86EMUL_CONTINUE;
3443 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3445 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3446 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3447 return X86EMUL_CONTINUE;
3450 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3452 switch (ctxt->op_bytes) {
3453 #ifdef CONFIG_X86_64
3455 asm("bswap %0" : "+r"(ctxt->dst.val));
3459 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3462 return X86EMUL_CONTINUE;
3465 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3467 /* emulating clflush regardless of cpuid */
3468 return X86EMUL_CONTINUE;
3471 static bool valid_cr(int nr)
3483 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3485 if (!valid_cr(ctxt->modrm_reg))
3486 return emulate_ud(ctxt);
3488 return X86EMUL_CONTINUE;
3491 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3493 u64 new_val = ctxt->src.val64;
3494 int cr = ctxt->modrm_reg;
3497 static u64 cr_reserved_bits[] = {
3498 0xffffffff00000000ULL,
3499 0, 0, 0, /* CR3 checked later */
3506 return emulate_ud(ctxt);
3508 if (new_val & cr_reserved_bits[cr])
3509 return emulate_gp(ctxt, 0);
3514 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3515 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3516 return emulate_gp(ctxt, 0);
3518 cr4 = ctxt->ops->get_cr(ctxt, 4);
3519 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3521 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3522 !(cr4 & X86_CR4_PAE))
3523 return emulate_gp(ctxt, 0);
3530 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3531 if (efer & EFER_LMA)
3532 rsvd = CR3_L_MODE_RESERVED_BITS;
3535 return emulate_gp(ctxt, 0);
3540 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3542 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3543 return emulate_gp(ctxt, 0);
3549 return X86EMUL_CONTINUE;
3552 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3556 ctxt->ops->get_dr(ctxt, 7, &dr7);
3558 /* Check if DR7.Global_Enable is set */
3559 return dr7 & (1 << 13);
3562 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3564 int dr = ctxt->modrm_reg;
3568 return emulate_ud(ctxt);
3570 cr4 = ctxt->ops->get_cr(ctxt, 4);
3571 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3572 return emulate_ud(ctxt);
3574 if (check_dr7_gd(ctxt))
3575 return emulate_db(ctxt);
3577 return X86EMUL_CONTINUE;
3580 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3582 u64 new_val = ctxt->src.val64;
3583 int dr = ctxt->modrm_reg;
3585 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3586 return emulate_gp(ctxt, 0);
3588 return check_dr_read(ctxt);
3591 static int check_svme(struct x86_emulate_ctxt *ctxt)
3595 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3597 if (!(efer & EFER_SVME))
3598 return emulate_ud(ctxt);
3600 return X86EMUL_CONTINUE;
3603 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3605 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3607 /* Valid physical address? */
3608 if (rax & 0xffff000000000000ULL)
3609 return emulate_gp(ctxt, 0);
3611 return check_svme(ctxt);
3614 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3616 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3618 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3619 return emulate_ud(ctxt);
3621 return X86EMUL_CONTINUE;
3624 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3626 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3627 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3629 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3630 ctxt->ops->check_pmc(ctxt, rcx))
3631 return emulate_gp(ctxt, 0);
3633 return X86EMUL_CONTINUE;
3636 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3638 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3639 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3640 return emulate_gp(ctxt, 0);
3642 return X86EMUL_CONTINUE;
3645 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3647 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3648 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3649 return emulate_gp(ctxt, 0);
3651 return X86EMUL_CONTINUE;
3654 #define D(_y) { .flags = (_y) }
3655 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3656 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3657 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3658 #define N D(NotImpl)
3659 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3660 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3661 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3662 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3663 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3664 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3665 #define II(_f, _e, _i) \
3666 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3667 #define IIP(_f, _e, _i, _p) \
3668 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3669 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3670 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3672 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3673 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3674 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3675 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3676 #define I2bvIP(_f, _e, _i, _p) \
3677 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3679 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3680 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3681 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3683 static const struct opcode group7_rm0[] = {
3685 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3689 static const struct opcode group7_rm1[] = {
3690 DI(SrcNone | Priv, monitor),
3691 DI(SrcNone | Priv, mwait),
3695 static const struct opcode group7_rm3[] = {
3696 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3697 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3698 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3699 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3700 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3701 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3702 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3703 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3706 static const struct opcode group7_rm7[] = {
3708 DIP(SrcNone, rdtscp, check_rdtsc),
3712 static const struct opcode group1[] = {
3714 F(Lock | PageTable, em_or),
3717 F(Lock | PageTable, em_and),
3723 static const struct opcode group1A[] = {
3724 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3727 static const struct opcode group2[] = {
3728 F(DstMem | ModRM, em_rol),
3729 F(DstMem | ModRM, em_ror),
3730 F(DstMem | ModRM, em_rcl),
3731 F(DstMem | ModRM, em_rcr),
3732 F(DstMem | ModRM, em_shl),
3733 F(DstMem | ModRM, em_shr),
3734 F(DstMem | ModRM, em_shl),
3735 F(DstMem | ModRM, em_sar),
3738 static const struct opcode group3[] = {
3739 F(DstMem | SrcImm | NoWrite, em_test),
3740 F(DstMem | SrcImm | NoWrite, em_test),
3741 F(DstMem | SrcNone | Lock, em_not),
3742 F(DstMem | SrcNone | Lock, em_neg),
3743 F(DstXacc | Src2Mem, em_mul_ex),
3744 F(DstXacc | Src2Mem, em_imul_ex),
3745 F(DstXacc | Src2Mem, em_div_ex),
3746 F(DstXacc | Src2Mem, em_idiv_ex),
3749 static const struct opcode group4[] = {
3750 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3751 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3755 static const struct opcode group5[] = {
3756 F(DstMem | SrcNone | Lock, em_inc),
3757 F(DstMem | SrcNone | Lock, em_dec),
3758 I(SrcMem | Stack, em_grp45),
3759 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3760 I(SrcMem | Stack, em_grp45),
3761 I(SrcMemFAddr | ImplicitOps, em_grp45),
3762 I(SrcMem | Stack, em_grp45), D(Undefined),
3765 static const struct opcode group6[] = {
3768 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3769 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3773 static const struct group_dual group7 = { {
3774 II(Mov | DstMem, em_sgdt, sgdt),
3775 II(Mov | DstMem, em_sidt, sidt),
3776 II(SrcMem | Priv, em_lgdt, lgdt),
3777 II(SrcMem | Priv, em_lidt, lidt),
3778 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3779 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3780 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3784 N, EXT(0, group7_rm3),
3785 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3786 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3790 static const struct opcode group8[] = {
3792 F(DstMem | SrcImmByte | NoWrite, em_bt),
3793 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3794 F(DstMem | SrcImmByte | Lock, em_btr),
3795 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3798 static const struct group_dual group9 = { {
3799 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3801 N, N, N, N, N, N, N, N,
3804 static const struct opcode group11[] = {
3805 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3809 static const struct gprefix pfx_0f_ae_7 = {
3810 I(SrcMem | ByteOp, em_clflush), N, N, N,
3813 static const struct group_dual group15 = { {
3814 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3816 N, N, N, N, N, N, N, N,
3819 static const struct gprefix pfx_0f_6f_0f_7f = {
3820 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3823 static const struct gprefix pfx_0f_2b = {
3824 I(0, em_mov), I(0, em_mov), N, N,
3827 static const struct gprefix pfx_0f_28_0f_29 = {
3828 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3831 static const struct gprefix pfx_0f_e7 = {
3832 N, I(Sse, em_mov), N, N,
3835 static const struct escape escape_d9 = { {
3836 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3839 N, N, N, N, N, N, N, N,
3841 N, N, N, N, N, N, N, N,
3843 N, N, N, N, N, N, N, N,
3845 N, N, N, N, N, N, N, N,
3847 N, N, N, N, N, N, N, N,
3849 N, N, N, N, N, N, N, N,
3851 N, N, N, N, N, N, N, N,
3853 N, N, N, N, N, N, N, N,
3856 static const struct escape escape_db = { {
3857 N, N, N, N, N, N, N, N,
3860 N, N, N, N, N, N, N, N,
3862 N, N, N, N, N, N, N, N,
3864 N, N, N, N, N, N, N, N,
3866 N, N, N, N, N, N, N, N,
3868 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3870 N, N, N, N, N, N, N, N,
3872 N, N, N, N, N, N, N, N,
3874 N, N, N, N, N, N, N, N,
3877 static const struct escape escape_dd = { {
3878 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3881 N, N, N, N, N, N, N, N,
3883 N, N, N, N, N, N, N, N,
3885 N, N, N, N, N, N, N, N,
3887 N, N, N, N, N, N, N, N,
3889 N, N, N, N, N, N, N, N,
3891 N, N, N, N, N, N, N, N,
3893 N, N, N, N, N, N, N, N,
3895 N, N, N, N, N, N, N, N,
3898 static const struct opcode opcode_table[256] = {
3900 F6ALU(Lock, em_add),
3901 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3902 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3904 F6ALU(Lock | PageTable, em_or),
3905 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3908 F6ALU(Lock, em_adc),
3909 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3910 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3912 F6ALU(Lock, em_sbb),
3913 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3914 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3916 F6ALU(Lock | PageTable, em_and), N, N,
3918 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3920 F6ALU(Lock, em_xor), N, N,
3922 F6ALU(NoWrite, em_cmp), N, N,
3924 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3926 X8(I(SrcReg | Stack, em_push)),
3928 X8(I(DstReg | Stack, em_pop)),
3930 I(ImplicitOps | Stack | No64, em_pusha),
3931 I(ImplicitOps | Stack | No64, em_popa),
3932 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3935 I(SrcImm | Mov | Stack, em_push),
3936 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3937 I(SrcImmByte | Mov | Stack, em_push),
3938 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3939 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3940 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3944 G(ByteOp | DstMem | SrcImm, group1),
3945 G(DstMem | SrcImm, group1),
3946 G(ByteOp | DstMem | SrcImm | No64, group1),
3947 G(DstMem | SrcImmByte, group1),
3948 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3949 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3951 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3952 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3953 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3954 D(ModRM | SrcMem | NoAccess | DstReg),
3955 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3958 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3960 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3961 I(SrcImmFAddr | No64, em_call_far), N,
3962 II(ImplicitOps | Stack, em_pushf, pushf),
3963 II(ImplicitOps | Stack, em_popf, popf),
3964 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3966 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3967 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3968 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3969 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3971 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3972 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3973 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3974 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3976 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3978 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3980 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3981 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3982 I(ImplicitOps | Stack, em_ret),
3983 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3984 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3985 G(ByteOp, group11), G(0, group11),
3987 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3988 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3989 I(ImplicitOps | Stack, em_ret_far),
3990 D(ImplicitOps), DI(SrcImmByte, intn),
3991 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3993 G(Src2One | ByteOp, group2), G(Src2One, group2),
3994 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3995 I(DstAcc | SrcImmUByte | No64, em_aam),
3996 I(DstAcc | SrcImmUByte | No64, em_aad),
3997 F(DstAcc | ByteOp | No64, em_salc),
3998 I(DstAcc | SrcXLat | ByteOp, em_mov),
4000 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4002 X3(I(SrcImmByte, em_loop)),
4003 I(SrcImmByte, em_jcxz),
4004 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4005 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4007 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
4008 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
4009 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4010 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4012 N, DI(ImplicitOps, icebp), N, N,
4013 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4014 G(ByteOp, group3), G(0, group3),
4016 D(ImplicitOps), D(ImplicitOps),
4017 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4018 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4021 static const struct opcode twobyte_table[256] = {
4023 G(0, group6), GD(0, &group7), N, N,
4024 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4025 II(ImplicitOps | Priv, em_clts, clts), N,
4026 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4027 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4029 N, N, N, N, N, N, N, N,
4030 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4031 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4033 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4034 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4035 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4037 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4040 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4041 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4042 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4045 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4046 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4047 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4048 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4049 I(ImplicitOps | EmulateOnUD, em_sysenter),
4050 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4052 N, N, N, N, N, N, N, N,
4054 X16(D(DstReg | SrcMem | ModRM)),
4056 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4061 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4066 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4070 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4072 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4073 II(ImplicitOps, em_cpuid, cpuid),
4074 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4075 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4076 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4078 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4079 DI(ImplicitOps, rsm),
4080 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4081 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4082 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4083 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4085 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4086 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4087 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4088 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4089 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4090 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4094 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4095 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4096 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4098 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4099 N, D(DstMem | SrcReg | ModRM | Mov),
4100 N, N, N, GD(0, &group9),
4102 X8(I(DstReg, em_bswap)),
4104 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4106 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4107 N, N, N, N, N, N, N, N,
4109 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4112 static const struct gprefix three_byte_0f_38_f0 = {
4113 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
4116 static const struct gprefix three_byte_0f_38_f1 = {
4117 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
4121 * Insns below are selected by the prefix which indexed by the third opcode
4124 static const struct opcode opcode_map_0f_38[256] = {
4126 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4128 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4130 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
4131 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4150 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4154 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4160 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4161 unsigned size, bool sign_extension)
4163 int rc = X86EMUL_CONTINUE;
4167 op->addr.mem.ea = ctxt->_eip;
4168 /* NB. Immediates are sign-extended as necessary. */
4169 switch (op->bytes) {
4171 op->val = insn_fetch(s8, ctxt);
4174 op->val = insn_fetch(s16, ctxt);
4177 op->val = insn_fetch(s32, ctxt);
4180 op->val = insn_fetch(s64, ctxt);
4183 if (!sign_extension) {
4184 switch (op->bytes) {
4192 op->val &= 0xffffffff;
4200 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4203 int rc = X86EMUL_CONTINUE;
4207 decode_register_operand(ctxt, op);
4210 rc = decode_imm(ctxt, op, 1, false);
4213 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4217 if (ctxt->d & BitOp)
4218 fetch_bit_operand(ctxt);
4219 op->orig_val = op->val;
4222 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4226 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4227 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4228 fetch_register_operand(op);
4229 op->orig_val = op->val;
4233 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4234 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4235 fetch_register_operand(op);
4236 op->orig_val = op->val;
4239 if (ctxt->d & ByteOp) {
4244 op->bytes = ctxt->op_bytes;
4245 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4246 fetch_register_operand(op);
4247 op->orig_val = op->val;
4251 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4253 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4254 op->addr.mem.seg = VCPU_SREG_ES;
4261 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4262 fetch_register_operand(op);
4266 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4269 rc = decode_imm(ctxt, op, 1, true);
4276 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4279 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4282 ctxt->memop.bytes = 1;
4283 if (ctxt->memop.type == OP_REG) {
4284 ctxt->memop.addr.reg = decode_register(ctxt,
4285 ctxt->modrm_rm, true);
4286 fetch_register_operand(&ctxt->memop);
4290 ctxt->memop.bytes = 2;
4293 ctxt->memop.bytes = 4;
4296 rc = decode_imm(ctxt, op, 2, false);
4299 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4303 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4305 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4306 op->addr.mem.seg = ctxt->seg_override;
4312 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4314 register_address(ctxt,
4315 reg_read(ctxt, VCPU_REGS_RBX) +
4316 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4317 op->addr.mem.seg = ctxt->seg_override;
4322 op->addr.mem.ea = ctxt->_eip;
4323 op->bytes = ctxt->op_bytes + 2;
4324 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4327 ctxt->memop.bytes = ctxt->op_bytes + 2;
4330 op->val = VCPU_SREG_ES;
4333 op->val = VCPU_SREG_CS;
4336 op->val = VCPU_SREG_SS;
4339 op->val = VCPU_SREG_DS;
4342 op->val = VCPU_SREG_FS;
4345 op->val = VCPU_SREG_GS;
4348 /* Special instructions do their own operand decoding. */
4350 op->type = OP_NONE; /* Disable writeback. */
4358 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4360 int rc = X86EMUL_CONTINUE;
4361 int mode = ctxt->mode;
4362 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4363 bool op_prefix = false;
4364 bool has_seg_override = false;
4365 struct opcode opcode;
4367 ctxt->memop.type = OP_NONE;
4368 ctxt->memopp = NULL;
4369 ctxt->_eip = ctxt->eip;
4370 ctxt->fetch.ptr = ctxt->fetch.data;
4371 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4372 ctxt->opcode_len = 1;
4374 memcpy(ctxt->fetch.data, insn, insn_len);
4376 rc = __do_insn_fetch_bytes(ctxt, 1);
4377 if (rc != X86EMUL_CONTINUE)
4382 case X86EMUL_MODE_REAL:
4383 case X86EMUL_MODE_VM86:
4384 case X86EMUL_MODE_PROT16:
4385 def_op_bytes = def_ad_bytes = 2;
4387 case X86EMUL_MODE_PROT32:
4388 def_op_bytes = def_ad_bytes = 4;
4390 #ifdef CONFIG_X86_64
4391 case X86EMUL_MODE_PROT64:
4397 return EMULATION_FAILED;
4400 ctxt->op_bytes = def_op_bytes;
4401 ctxt->ad_bytes = def_ad_bytes;
4403 /* Legacy prefixes. */
4405 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4406 case 0x66: /* operand-size override */
4408 /* switch between 2/4 bytes */
4409 ctxt->op_bytes = def_op_bytes ^ 6;
4411 case 0x67: /* address-size override */
4412 if (mode == X86EMUL_MODE_PROT64)
4413 /* switch between 4/8 bytes */
4414 ctxt->ad_bytes = def_ad_bytes ^ 12;
4416 /* switch between 2/4 bytes */
4417 ctxt->ad_bytes = def_ad_bytes ^ 6;
4419 case 0x26: /* ES override */
4420 case 0x2e: /* CS override */
4421 case 0x36: /* SS override */
4422 case 0x3e: /* DS override */
4423 has_seg_override = true;
4424 ctxt->seg_override = (ctxt->b >> 3) & 3;
4426 case 0x64: /* FS override */
4427 case 0x65: /* GS override */
4428 has_seg_override = true;
4429 ctxt->seg_override = ctxt->b & 7;
4431 case 0x40 ... 0x4f: /* REX */
4432 if (mode != X86EMUL_MODE_PROT64)
4434 ctxt->rex_prefix = ctxt->b;
4436 case 0xf0: /* LOCK */
4437 ctxt->lock_prefix = 1;
4439 case 0xf2: /* REPNE/REPNZ */
4440 case 0xf3: /* REP/REPE/REPZ */
4441 ctxt->rep_prefix = ctxt->b;
4447 /* Any legacy prefix after a REX prefix nullifies its effect. */
4449 ctxt->rex_prefix = 0;
4455 if (ctxt->rex_prefix & 8)
4456 ctxt->op_bytes = 8; /* REX.W */
4458 /* Opcode byte(s). */
4459 opcode = opcode_table[ctxt->b];
4460 /* Two-byte opcode? */
4461 if (ctxt->b == 0x0f) {
4462 ctxt->opcode_len = 2;
4463 ctxt->b = insn_fetch(u8, ctxt);
4464 opcode = twobyte_table[ctxt->b];
4466 /* 0F_38 opcode map */
4467 if (ctxt->b == 0x38) {
4468 ctxt->opcode_len = 3;
4469 ctxt->b = insn_fetch(u8, ctxt);
4470 opcode = opcode_map_0f_38[ctxt->b];
4473 ctxt->d = opcode.flags;
4475 if (ctxt->d & ModRM)
4476 ctxt->modrm = insn_fetch(u8, ctxt);
4478 /* vex-prefix instructions are not implemented */
4479 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4480 (mode == X86EMUL_MODE_PROT64 ||
4481 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4485 while (ctxt->d & GroupMask) {
4486 switch (ctxt->d & GroupMask) {
4488 goffset = (ctxt->modrm >> 3) & 7;
4489 opcode = opcode.u.group[goffset];
4492 goffset = (ctxt->modrm >> 3) & 7;
4493 if ((ctxt->modrm >> 6) == 3)
4494 opcode = opcode.u.gdual->mod3[goffset];
4496 opcode = opcode.u.gdual->mod012[goffset];
4499 goffset = ctxt->modrm & 7;
4500 opcode = opcode.u.group[goffset];
4503 if (ctxt->rep_prefix && op_prefix)
4504 return EMULATION_FAILED;
4505 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4506 switch (simd_prefix) {
4507 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4508 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4509 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4510 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4514 if (ctxt->modrm > 0xbf)
4515 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4517 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4520 return EMULATION_FAILED;
4523 ctxt->d &= ~(u64)GroupMask;
4524 ctxt->d |= opcode.flags;
4529 return EMULATION_FAILED;
4531 ctxt->execute = opcode.u.execute;
4533 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4534 return EMULATION_FAILED;
4536 if (unlikely(ctxt->d &
4537 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4539 * These are copied unconditionally here, and checked unconditionally
4540 * in x86_emulate_insn.
4542 ctxt->check_perm = opcode.check_perm;
4543 ctxt->intercept = opcode.intercept;
4545 if (ctxt->d & NotImpl)
4546 return EMULATION_FAILED;
4548 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4551 if (ctxt->d & Op3264) {
4552 if (mode == X86EMUL_MODE_PROT64)
4559 ctxt->op_bytes = 16;
4560 else if (ctxt->d & Mmx)
4564 /* ModRM and SIB bytes. */
4565 if (ctxt->d & ModRM) {
4566 rc = decode_modrm(ctxt, &ctxt->memop);
4567 if (!has_seg_override) {
4568 has_seg_override = true;
4569 ctxt->seg_override = ctxt->modrm_seg;
4571 } else if (ctxt->d & MemAbs)
4572 rc = decode_abs(ctxt, &ctxt->memop);
4573 if (rc != X86EMUL_CONTINUE)
4576 if (!has_seg_override)
4577 ctxt->seg_override = VCPU_SREG_DS;
4579 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4582 * Decode and fetch the source operand: register, memory
4585 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4586 if (rc != X86EMUL_CONTINUE)
4590 * Decode and fetch the second source operand: register, memory
4593 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4594 if (rc != X86EMUL_CONTINUE)
4597 /* Decode and fetch the destination operand: register or memory. */
4598 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4600 if (ctxt->rip_relative)
4601 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4604 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4607 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4609 return ctxt->d & PageTable;
4612 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4614 /* The second termination condition only applies for REPE
4615 * and REPNE. Test if the repeat string operation prefix is
4616 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4617 * corresponding termination condition according to:
4618 * - if REPE/REPZ and ZF = 0 then done
4619 * - if REPNE/REPNZ and ZF = 1 then done
4621 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4622 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4623 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4624 ((ctxt->eflags & EFLG_ZF) == 0))
4625 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4626 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4632 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4636 ctxt->ops->get_fpu(ctxt);
4637 asm volatile("1: fwait \n\t"
4639 ".pushsection .fixup,\"ax\" \n\t"
4641 "movb $1, %[fault] \n\t"
4644 _ASM_EXTABLE(1b, 3b)
4645 : [fault]"+qm"(fault));
4646 ctxt->ops->put_fpu(ctxt);
4648 if (unlikely(fault))
4649 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4651 return X86EMUL_CONTINUE;
4654 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4657 if (op->type == OP_MM)
4658 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4661 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4663 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4664 if (!(ctxt->d & ByteOp))
4665 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4666 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4667 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4669 : "c"(ctxt->src2.val));
4670 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4671 if (!fop) /* exception is returned in fop variable */
4672 return emulate_de(ctxt);
4673 return X86EMUL_CONTINUE;
4676 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4678 memset(&ctxt->rip_relative, 0,
4679 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4681 ctxt->io_read.pos = 0;
4682 ctxt->io_read.end = 0;
4683 ctxt->mem_read.end = 0;
4686 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4688 const struct x86_emulate_ops *ops = ctxt->ops;
4689 int rc = X86EMUL_CONTINUE;
4690 int saved_dst_type = ctxt->dst.type;
4692 ctxt->mem_read.pos = 0;
4694 /* LOCK prefix is allowed only with some instructions */
4695 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4696 rc = emulate_ud(ctxt);
4700 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4701 rc = emulate_ud(ctxt);
4705 if (unlikely(ctxt->d &
4706 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4707 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4708 (ctxt->d & Undefined)) {
4709 rc = emulate_ud(ctxt);
4713 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4714 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4715 rc = emulate_ud(ctxt);
4719 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4720 rc = emulate_nm(ctxt);
4724 if (ctxt->d & Mmx) {
4725 rc = flush_pending_x87_faults(ctxt);
4726 if (rc != X86EMUL_CONTINUE)
4729 * Now that we know the fpu is exception safe, we can fetch
4732 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4733 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4734 if (!(ctxt->d & Mov))
4735 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4738 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4739 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4740 X86_ICPT_PRE_EXCEPT);
4741 if (rc != X86EMUL_CONTINUE)
4745 /* Privileged instruction can be executed only in CPL=0 */
4746 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4747 if (ctxt->d & PrivUD)
4748 rc = emulate_ud(ctxt);
4750 rc = emulate_gp(ctxt, 0);
4754 /* Instruction can only be executed in protected mode */
4755 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4756 rc = emulate_ud(ctxt);
4760 /* Do instruction specific permission checks */
4761 if (ctxt->d & CheckPerm) {
4762 rc = ctxt->check_perm(ctxt);
4763 if (rc != X86EMUL_CONTINUE)
4767 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4768 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4769 X86_ICPT_POST_EXCEPT);
4770 if (rc != X86EMUL_CONTINUE)
4774 if (ctxt->rep_prefix && (ctxt->d & String)) {
4775 /* All REP prefixes have the same first termination condition */
4776 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4777 ctxt->eip = ctxt->_eip;
4778 ctxt->eflags &= ~EFLG_RF;
4784 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4785 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4786 ctxt->src.valptr, ctxt->src.bytes);
4787 if (rc != X86EMUL_CONTINUE)
4789 ctxt->src.orig_val64 = ctxt->src.val64;
4792 if (ctxt->src2.type == OP_MEM) {
4793 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4794 &ctxt->src2.val, ctxt->src2.bytes);
4795 if (rc != X86EMUL_CONTINUE)
4799 if ((ctxt->d & DstMask) == ImplicitOps)
4803 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4804 /* optimisation - avoid slow emulated read if Mov */
4805 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4806 &ctxt->dst.val, ctxt->dst.bytes);
4807 if (rc != X86EMUL_CONTINUE)
4810 ctxt->dst.orig_val = ctxt->dst.val;
4814 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4815 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4816 X86_ICPT_POST_MEMACCESS);
4817 if (rc != X86EMUL_CONTINUE)
4821 if (ctxt->rep_prefix && (ctxt->d & String))
4822 ctxt->eflags |= EFLG_RF;
4824 ctxt->eflags &= ~EFLG_RF;
4826 if (ctxt->execute) {
4827 if (ctxt->d & Fastop) {
4828 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4829 rc = fastop(ctxt, fop);
4830 if (rc != X86EMUL_CONTINUE)
4834 rc = ctxt->execute(ctxt);
4835 if (rc != X86EMUL_CONTINUE)
4840 if (ctxt->opcode_len == 2)
4842 else if (ctxt->opcode_len == 3)
4843 goto threebyte_insn;
4846 case 0x63: /* movsxd */
4847 if (ctxt->mode != X86EMUL_MODE_PROT64)
4848 goto cannot_emulate;
4849 ctxt->dst.val = (s32) ctxt->src.val;
4851 case 0x70 ... 0x7f: /* jcc (short) */
4852 if (test_cc(ctxt->b, ctxt->eflags))
4853 rc = jmp_rel(ctxt, ctxt->src.val);
4855 case 0x8d: /* lea r16/r32, m */
4856 ctxt->dst.val = ctxt->src.addr.mem.ea;
4858 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4859 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4860 ctxt->dst.type = OP_NONE;
4864 case 0x98: /* cbw/cwde/cdqe */
4865 switch (ctxt->op_bytes) {
4866 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4867 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4868 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4871 case 0xcc: /* int3 */
4872 rc = emulate_int(ctxt, 3);
4874 case 0xcd: /* int n */
4875 rc = emulate_int(ctxt, ctxt->src.val);
4877 case 0xce: /* into */
4878 if (ctxt->eflags & EFLG_OF)
4879 rc = emulate_int(ctxt, 4);
4881 case 0xe9: /* jmp rel */
4882 case 0xeb: /* jmp rel short */
4883 rc = jmp_rel(ctxt, ctxt->src.val);
4884 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4886 case 0xf4: /* hlt */
4887 ctxt->ops->halt(ctxt);
4889 case 0xf5: /* cmc */
4890 /* complement carry flag from eflags reg */
4891 ctxt->eflags ^= EFLG_CF;
4893 case 0xf8: /* clc */
4894 ctxt->eflags &= ~EFLG_CF;
4896 case 0xf9: /* stc */
4897 ctxt->eflags |= EFLG_CF;
4899 case 0xfc: /* cld */
4900 ctxt->eflags &= ~EFLG_DF;
4902 case 0xfd: /* std */
4903 ctxt->eflags |= EFLG_DF;
4906 goto cannot_emulate;
4909 if (rc != X86EMUL_CONTINUE)
4913 if (ctxt->d & SrcWrite) {
4914 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4915 rc = writeback(ctxt, &ctxt->src);
4916 if (rc != X86EMUL_CONTINUE)
4919 if (!(ctxt->d & NoWrite)) {
4920 rc = writeback(ctxt, &ctxt->dst);
4921 if (rc != X86EMUL_CONTINUE)
4926 * restore dst type in case the decoding will be reused
4927 * (happens for string instruction )
4929 ctxt->dst.type = saved_dst_type;
4931 if ((ctxt->d & SrcMask) == SrcSI)
4932 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4934 if ((ctxt->d & DstMask) == DstDI)
4935 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4937 if (ctxt->rep_prefix && (ctxt->d & String)) {
4939 struct read_cache *r = &ctxt->io_read;
4940 if ((ctxt->d & SrcMask) == SrcSI)
4941 count = ctxt->src.count;
4943 count = ctxt->dst.count;
4944 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4947 if (!string_insn_completed(ctxt)) {
4949 * Re-enter guest when pio read ahead buffer is empty
4950 * or, if it is not used, after each 1024 iteration.
4952 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4953 (r->end == 0 || r->end != r->pos)) {
4955 * Reset read cache. Usually happens before
4956 * decode, but since instruction is restarted
4957 * we have to do it here.
4959 ctxt->mem_read.end = 0;
4960 writeback_registers(ctxt);
4961 return EMULATION_RESTART;
4963 goto done; /* skip rip writeback */
4965 ctxt->eflags &= ~EFLG_RF;
4968 ctxt->eip = ctxt->_eip;
4971 if (rc == X86EMUL_PROPAGATE_FAULT) {
4972 WARN_ON(ctxt->exception.vector > 0x1f);
4973 ctxt->have_exception = true;
4975 if (rc == X86EMUL_INTERCEPTED)
4976 return EMULATION_INTERCEPTED;
4978 if (rc == X86EMUL_CONTINUE)
4979 writeback_registers(ctxt);
4981 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4985 case 0x09: /* wbinvd */
4986 (ctxt->ops->wbinvd)(ctxt);
4988 case 0x08: /* invd */
4989 case 0x0d: /* GrpP (prefetch) */
4990 case 0x18: /* Grp16 (prefetch/nop) */
4991 case 0x1f: /* nop */
4993 case 0x20: /* mov cr, reg */
4994 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4996 case 0x21: /* mov from dr to reg */
4997 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4999 case 0x40 ... 0x4f: /* cmov */
5000 if (test_cc(ctxt->b, ctxt->eflags))
5001 ctxt->dst.val = ctxt->src.val;
5002 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
5003 ctxt->op_bytes != 4)
5004 ctxt->dst.type = OP_NONE; /* no writeback */
5006 case 0x80 ... 0x8f: /* jnz rel, etc*/
5007 if (test_cc(ctxt->b, ctxt->eflags))
5008 rc = jmp_rel(ctxt, ctxt->src.val);
5010 case 0x90 ... 0x9f: /* setcc r/m8 */
5011 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5013 case 0xb6 ... 0xb7: /* movzx */
5014 ctxt->dst.bytes = ctxt->op_bytes;
5015 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5016 : (u16) ctxt->src.val;
5018 case 0xbe ... 0xbf: /* movsx */
5019 ctxt->dst.bytes = ctxt->op_bytes;
5020 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5021 (s16) ctxt->src.val;
5023 case 0xc3: /* movnti */
5024 ctxt->dst.bytes = ctxt->op_bytes;
5025 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
5026 (u32) ctxt->src.val;
5029 goto cannot_emulate;
5034 if (rc != X86EMUL_CONTINUE)
5040 return EMULATION_FAILED;
5043 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5045 invalidate_registers(ctxt);
5048 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5050 writeback_registers(ctxt);