1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affilates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
26 #include <public/xen.h>
27 #define DPRINTF(_f, _a ...) printf(_f , ## _a)
29 #include <linux/kvm_host.h>
30 #include "kvm_cache_regs.h"
31 #define DPRINTF(x...) do {} while (0)
33 #include <linux/module.h>
34 #include <asm/kvm_emulate.h>
40 * Opcode effective-address decode tables.
41 * Note that we only emulate instructions that have at least one memory
42 * operand (excluding implicit stack references). We assume that stack
43 * references and instruction fetches will never occur in special memory
44 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
48 /* Operand sizes: 8-bit operands or specified/overridden size. */
49 #define ByteOp (1<<16) /* 8-bit operands. */
50 /* Destination operand type. */
51 #define ImplicitOps (1<<17) /* Implicit in opcode. No generic decode. */
52 #define DstReg (2<<17) /* Register operand. */
53 #define DstMem (3<<17) /* Memory operand. */
54 #define DstAcc (4<<17) /* Destination Accumulator */
55 #define DstDI (5<<17) /* Destination is in ES:(E)DI */
56 #define DstMem64 (6<<17) /* 64bit memory operand */
57 #define DstMask (7<<17)
58 /* Source operand type. */
59 #define SrcNone (0<<4) /* No source operand. */
60 #define SrcImplicit (0<<4) /* Source operand is implicit in the opcode. */
61 #define SrcReg (1<<4) /* Register operand. */
62 #define SrcMem (2<<4) /* Memory operand. */
63 #define SrcMem16 (3<<4) /* Memory operand (16-bit). */
64 #define SrcMem32 (4<<4) /* Memory operand (32-bit). */
65 #define SrcImm (5<<4) /* Immediate operand. */
66 #define SrcImmByte (6<<4) /* 8-bit sign-extended immediate operand. */
67 #define SrcOne (7<<4) /* Implied '1' */
68 #define SrcImmUByte (8<<4) /* 8-bit unsigned immediate operand. */
69 #define SrcImmU (9<<4) /* Immediate operand, unsigned */
70 #define SrcSI (0xa<<4) /* Source is in the DS:RSI */
71 #define SrcImmFAddr (0xb<<4) /* Source is immediate far address */
72 #define SrcMemFAddr (0xc<<4) /* Source is far address in memory */
73 #define SrcAcc (0xd<<4) /* Source Accumulator */
74 #define SrcMask (0xf<<4)
75 /* Generic ModRM decode. */
77 /* Destination is only written; never read. */
80 #define MemAbs (1<<11) /* Memory operand is absolute displacement */
81 #define String (1<<12) /* String instruction (rep capable) */
82 #define Stack (1<<13) /* Stack instruction (push/pop) */
83 #define Group (1<<14) /* Bits 3:5 of modrm byte extend opcode */
84 #define GroupDual (1<<15) /* Alternate decoding of mod == 3 */
85 #define GroupMask 0x0f /* Group number stored in bits 0:3 */
87 #define Undefined (1<<25) /* No Such Instruction */
88 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
89 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
91 /* Source 2 operand type */
92 #define Src2None (0<<29)
93 #define Src2CL (1<<29)
94 #define Src2ImmByte (2<<29)
95 #define Src2One (3<<29)
96 #define Src2Mask (7<<29)
99 #define X3(x) X2(x), x
100 #define X4(x) X2(x), X2(x)
101 #define X5(x) X4(x), x
102 #define X6(x) X4(x), X2(x)
103 #define X7(x) X4(x), X3(x)
104 #define X8(x) X4(x), X4(x)
105 #define X16(x) X8(x), X8(x)
108 NoGrp, Group1A, Group3, Group4, Group5, Group7, Group8, Group9,
114 struct opcode *group;
115 struct group_dual *gdual;
120 struct opcode mod012[8];
121 struct opcode mod3[8];
124 #define D(_y) { .flags = (_y) }
126 #define G(_f, _g) { .flags = ((_f) | Group), .u.group = (_g) }
127 #define GD(_f, _g) { .flags = ((_f) | Group | GroupDual), .u.gdual = (_g) }
129 static struct opcode group1[] = {
133 static struct opcode group_table[] = {
135 D(DstMem | SrcNone | ModRM | Mov | Stack), N, N, N, N, N, N, N,
137 D(DstMem | SrcImm | ModRM), D(DstMem | SrcImm | ModRM),
138 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
141 D(ByteOp | DstMem | SrcNone | ModRM | Lock), D(ByteOp | DstMem | SrcNone | ModRM | Lock),
144 D(DstMem | SrcNone | ModRM | Lock), D(DstMem | SrcNone | ModRM | Lock),
145 D(SrcMem | ModRM | Stack), N,
146 D(SrcMem | ModRM | Stack), D(SrcMemFAddr | ModRM | ImplicitOps),
147 D(SrcMem | ModRM | Stack), N,
149 N, N, D(ModRM | SrcMem | Priv), D(ModRM | SrcMem | Priv),
150 D(SrcNone | ModRM | DstMem | Mov), N,
151 D(SrcMem16 | ModRM | Mov | Priv), D(SrcMem | ModRM | ByteOp | Priv),
154 D(DstMem | SrcImmByte | ModRM), D(DstMem | SrcImmByte | ModRM | Lock),
155 D(DstMem | SrcImmByte | ModRM | Lock), D(DstMem | SrcImmByte | ModRM | Lock),
157 N, D(DstMem64 | ModRM | Lock), N, N, N, N, N, N,
160 static struct opcode group2_table[] = {
162 D(SrcNone | ModRM | Priv), N, N, D(SrcNone | ModRM | Priv),
163 D(SrcNone | ModRM | DstMem | Mov), N,
164 D(SrcMem16 | ModRM | Mov | Priv), N,
166 N, N, N, N, N, N, N, N,
169 static struct opcode opcode_table[256] = {
171 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
172 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
173 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
174 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
176 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
177 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
178 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
179 D(ImplicitOps | Stack | No64), N,
181 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
182 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
183 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
184 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
186 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
187 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
188 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
189 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
191 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
192 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
193 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
195 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
196 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
197 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
199 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
200 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
201 D(ByteOp | DstAcc | SrcImmByte), D(DstAcc | SrcImm), N, N,
203 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
204 D(ByteOp | DstReg | SrcMem | ModRM), D(DstReg | SrcMem | ModRM),
205 D(ByteOp | DstAcc | SrcImm), D(DstAcc | SrcImm),
210 X8(D(SrcReg | Stack)),
212 X8(D(DstReg | Stack)),
214 D(ImplicitOps | Stack | No64), D(ImplicitOps | Stack | No64),
215 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
218 D(SrcImm | Mov | Stack), N, D(SrcImmByte | Mov | Stack), N,
219 D(DstDI | ByteOp | Mov | String), D(DstDI | Mov | String), /* insb, insw/insd */
220 D(SrcSI | ByteOp | ImplicitOps | String), D(SrcSI | ImplicitOps | String), /* outsb, outsw/outsd */
224 G(ByteOp | DstMem | SrcImm | ModRM | Group, group1),
225 G(DstMem | SrcImm | ModRM | Group, group1),
226 G(ByteOp | DstMem | SrcImm | ModRM | No64 | Group, group1),
227 G(DstMem | SrcImmByte | ModRM | Group, group1),
228 D(ByteOp | DstMem | SrcReg | ModRM), D(DstMem | SrcReg | ModRM),
229 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
231 D(ByteOp | DstMem | SrcReg | ModRM | Mov), D(DstMem | SrcReg | ModRM | Mov),
232 D(ByteOp | DstReg | SrcMem | ModRM | Mov), D(DstReg | SrcMem | ModRM | Mov),
233 D(DstMem | SrcNone | ModRM | Mov), D(ModRM | DstReg),
234 D(ImplicitOps | SrcMem16 | ModRM), D(Group | Group1A),
236 D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg), D(DstReg),
238 N, N, D(SrcImmFAddr | No64), N,
239 D(ImplicitOps | Stack), D(ImplicitOps | Stack), N, N,
241 D(ByteOp | DstAcc | SrcMem | Mov | MemAbs), D(DstAcc | SrcMem | Mov | MemAbs),
242 D(ByteOp | DstMem | SrcAcc | Mov | MemAbs), D(DstMem | SrcAcc | Mov | MemAbs),
243 D(ByteOp | SrcSI | DstDI | Mov | String), D(SrcSI | DstDI | Mov | String),
244 D(ByteOp | SrcSI | DstDI | String), D(SrcSI | DstDI | String),
246 D(DstAcc | SrcImmByte | ByteOp), D(DstAcc | SrcImm), D(ByteOp | DstDI | Mov | String), D(DstDI | Mov | String),
247 D(ByteOp | SrcSI | DstAcc | Mov | String), D(SrcSI | DstAcc | Mov | String),
248 D(ByteOp | DstDI | String), D(DstDI | String),
250 X8(D(ByteOp | DstReg | SrcImm | Mov)),
252 X8(D(DstReg | SrcImm | Mov)),
254 D(ByteOp | DstMem | SrcImm | ModRM), D(DstMem | SrcImmByte | ModRM),
255 N, D(ImplicitOps | Stack), N, N,
256 D(ByteOp | DstMem | SrcImm | ModRM | Mov), D(DstMem | SrcImm | ModRM | Mov),
258 N, N, N, D(ImplicitOps | Stack),
259 D(ImplicitOps), D(SrcImmByte), D(ImplicitOps | No64), D(ImplicitOps),
261 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
262 D(ByteOp | DstMem | SrcImplicit | ModRM), D(DstMem | SrcImplicit | ModRM),
265 N, N, N, N, N, N, N, N,
268 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
269 D(ByteOp | SrcImmUByte | DstAcc), D(SrcImmUByte | DstAcc),
271 D(SrcImm | Stack), D(SrcImm | ImplicitOps),
272 D(SrcImmFAddr | No64), D(SrcImmByte | ImplicitOps),
273 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
274 D(SrcNone | ByteOp | DstAcc), D(SrcNone | DstAcc),
277 D(ImplicitOps | Priv), D(ImplicitOps), D(ByteOp | Group | Group3), D(Group | Group3),
279 D(ImplicitOps), N, D(ImplicitOps), D(ImplicitOps),
280 D(ImplicitOps), D(ImplicitOps), D(Group | Group4), D(Group | Group5),
283 static struct opcode twobyte_table[256] = {
285 N, D(Group | GroupDual | Group7), N, N,
286 N, D(ImplicitOps), D(ImplicitOps | Priv), N,
287 D(ImplicitOps | Priv), D(ImplicitOps | Priv), N, N,
288 N, D(ImplicitOps | ModRM), N, N,
290 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
292 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
293 D(ModRM | ImplicitOps | Priv), D(ModRM | Priv),
295 N, N, N, N, N, N, N, N,
297 D(ImplicitOps | Priv), N, D(ImplicitOps | Priv), N,
298 D(ImplicitOps), D(ImplicitOps | Priv), N, N,
299 N, N, N, N, N, N, N, N,
301 X16(D(DstReg | SrcMem | ModRM | Mov)),
303 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
305 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
307 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
311 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
313 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
314 N, D(DstMem | SrcReg | ModRM | BitOp),
315 D(DstMem | SrcReg | Src2ImmByte | ModRM),
316 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
318 D(ImplicitOps | Stack), D(ImplicitOps | Stack),
319 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
320 D(DstMem | SrcReg | Src2ImmByte | ModRM),
321 D(DstMem | SrcReg | Src2CL | ModRM),
324 D(ByteOp | DstMem | SrcReg | ModRM | Lock), D(DstMem | SrcReg | ModRM | Lock),
325 N, D(DstMem | SrcReg | ModRM | BitOp | Lock),
326 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
327 D(DstReg | SrcMem16 | ModRM | Mov),
330 D(Group | Group8), D(DstMem | SrcReg | ModRM | BitOp | Lock),
331 N, N, D(ByteOp | DstReg | SrcMem | ModRM | Mov),
332 D(DstReg | SrcMem16 | ModRM | Mov),
334 N, N, N, D(DstMem | SrcReg | ModRM | Mov),
335 N, N, N, D(Group | GroupDual | Group9),
336 N, N, N, N, N, N, N, N,
338 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
340 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
342 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
350 /* EFLAGS bit definitions. */
351 #define EFLG_ID (1<<21)
352 #define EFLG_VIP (1<<20)
353 #define EFLG_VIF (1<<19)
354 #define EFLG_AC (1<<18)
355 #define EFLG_VM (1<<17)
356 #define EFLG_RF (1<<16)
357 #define EFLG_IOPL (3<<12)
358 #define EFLG_NT (1<<14)
359 #define EFLG_OF (1<<11)
360 #define EFLG_DF (1<<10)
361 #define EFLG_IF (1<<9)
362 #define EFLG_TF (1<<8)
363 #define EFLG_SF (1<<7)
364 #define EFLG_ZF (1<<6)
365 #define EFLG_AF (1<<4)
366 #define EFLG_PF (1<<2)
367 #define EFLG_CF (1<<0)
369 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
370 #define EFLG_RESERVED_ONE_MASK 2
373 * Instruction emulation:
374 * Most instructions are emulated directly via a fragment of inline assembly
375 * code. This allows us to save/restore EFLAGS and thus very easily pick up
376 * any modified flags.
379 #if defined(CONFIG_X86_64)
380 #define _LO32 "k" /* force 32-bit operand */
381 #define _STK "%%rsp" /* stack pointer */
382 #elif defined(__i386__)
383 #define _LO32 "" /* force 32-bit operand */
384 #define _STK "%%esp" /* stack pointer */
388 * These EFLAGS bits are restored from saved value during emulation, and
389 * any changes are written back to the saved value after emulation.
391 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
393 /* Before executing instruction: restore necessary bits in EFLAGS. */
394 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
395 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
396 "movl %"_sav",%"_LO32 _tmp"; " \
399 "movl %"_msk",%"_LO32 _tmp"; " \
400 "andl %"_LO32 _tmp",("_STK"); " \
402 "notl %"_LO32 _tmp"; " \
403 "andl %"_LO32 _tmp",("_STK"); " \
404 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
406 "orl %"_LO32 _tmp",("_STK"); " \
410 /* After executing instruction: write-back necessary bits in EFLAGS. */
411 #define _POST_EFLAGS(_sav, _msk, _tmp) \
412 /* _sav |= EFLAGS & _msk; */ \
415 "andl %"_msk",%"_LO32 _tmp"; " \
416 "orl %"_LO32 _tmp",%"_sav"; "
424 #define ____emulate_2op(_op, _src, _dst, _eflags, _x, _y, _suffix) \
426 __asm__ __volatile__ ( \
427 _PRE_EFLAGS("0", "4", "2") \
428 _op _suffix " %"_x"3,%1; " \
429 _POST_EFLAGS("0", "4", "2") \
430 : "=m" (_eflags), "=m" ((_dst).val), \
432 : _y ((_src).val), "i" (EFLAGS_MASK)); \
436 /* Raw emulation: instruction has two explicit operands. */
437 #define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \
439 unsigned long _tmp; \
441 switch ((_dst).bytes) { \
443 ____emulate_2op(_op,_src,_dst,_eflags,_wx,_wy,"w"); \
446 ____emulate_2op(_op,_src,_dst,_eflags,_lx,_ly,"l"); \
449 ON64(____emulate_2op(_op,_src,_dst,_eflags,_qx,_qy,"q")); \
454 #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
456 unsigned long _tmp; \
457 switch ((_dst).bytes) { \
459 ____emulate_2op(_op,_src,_dst,_eflags,_bx,_by,"b"); \
462 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
463 _wx, _wy, _lx, _ly, _qx, _qy); \
468 /* Source operand is byte-sized and may be restricted to just %cl. */
469 #define emulate_2op_SrcB(_op, _src, _dst, _eflags) \
470 __emulate_2op(_op, _src, _dst, _eflags, \
471 "b", "c", "b", "c", "b", "c", "b", "c")
473 /* Source operand is byte, word, long or quad sized. */
474 #define emulate_2op_SrcV(_op, _src, _dst, _eflags) \
475 __emulate_2op(_op, _src, _dst, _eflags, \
476 "b", "q", "w", "r", _LO32, "r", "", "r")
478 /* Source operand is word, long or quad sized. */
479 #define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \
480 __emulate_2op_nobyte(_op, _src, _dst, _eflags, \
481 "w", "r", _LO32, "r", "", "r")
483 /* Instruction has three operands and one operand is stored in ECX register */
484 #define __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, _suffix, _type) \
486 unsigned long _tmp; \
487 _type _clv = (_cl).val; \
488 _type _srcv = (_src).val; \
489 _type _dstv = (_dst).val; \
491 __asm__ __volatile__ ( \
492 _PRE_EFLAGS("0", "5", "2") \
493 _op _suffix " %4,%1 \n" \
494 _POST_EFLAGS("0", "5", "2") \
495 : "=m" (_eflags), "+r" (_dstv), "=&r" (_tmp) \
496 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
499 (_cl).val = (unsigned long) _clv; \
500 (_src).val = (unsigned long) _srcv; \
501 (_dst).val = (unsigned long) _dstv; \
504 #define emulate_2op_cl(_op, _cl, _src, _dst, _eflags) \
506 switch ((_dst).bytes) { \
508 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
509 "w", unsigned short); \
512 __emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
513 "l", unsigned int); \
516 ON64(__emulate_2op_cl(_op, _cl, _src, _dst, _eflags, \
517 "q", unsigned long)); \
522 #define __emulate_1op(_op, _dst, _eflags, _suffix) \
524 unsigned long _tmp; \
526 __asm__ __volatile__ ( \
527 _PRE_EFLAGS("0", "3", "2") \
528 _op _suffix " %1; " \
529 _POST_EFLAGS("0", "3", "2") \
530 : "=m" (_eflags), "+m" ((_dst).val), \
532 : "i" (EFLAGS_MASK)); \
535 /* Instruction has only one explicit operand (no source operand). */
536 #define emulate_1op(_op, _dst, _eflags) \
538 switch ((_dst).bytes) { \
539 case 1: __emulate_1op(_op, _dst, _eflags, "b"); break; \
540 case 2: __emulate_1op(_op, _dst, _eflags, "w"); break; \
541 case 4: __emulate_1op(_op, _dst, _eflags, "l"); break; \
542 case 8: ON64(__emulate_1op(_op, _dst, _eflags, "q")); break; \
546 /* Fetch next part of the instruction being emulated. */
547 #define insn_fetch(_type, _size, _eip) \
548 ({ unsigned long _x; \
549 rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \
550 if (rc != X86EMUL_CONTINUE) \
556 #define insn_fetch_arr(_arr, _size, _eip) \
557 ({ rc = do_insn_fetch(ctxt, ops, (_eip), _arr, (_size)); \
558 if (rc != X86EMUL_CONTINUE) \
563 static inline unsigned long ad_mask(struct decode_cache *c)
565 return (1UL << (c->ad_bytes << 3)) - 1;
568 /* Access/update address held in a register, based on addressing mode. */
569 static inline unsigned long
570 address_mask(struct decode_cache *c, unsigned long reg)
572 if (c->ad_bytes == sizeof(unsigned long))
575 return reg & ad_mask(c);
578 static inline unsigned long
579 register_address(struct decode_cache *c, unsigned long base, unsigned long reg)
581 return base + address_mask(c, reg);
585 register_address_increment(struct decode_cache *c, unsigned long *reg, int inc)
587 if (c->ad_bytes == sizeof(unsigned long))
590 *reg = (*reg & ~ad_mask(c)) | ((*reg + inc) & ad_mask(c));
593 static inline void jmp_rel(struct decode_cache *c, int rel)
595 register_address_increment(c, &c->eip, rel);
598 static void set_seg_override(struct decode_cache *c, int seg)
600 c->has_seg_override = true;
601 c->seg_override = seg;
604 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt,
605 struct x86_emulate_ops *ops, int seg)
607 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
610 return ops->get_cached_segment_base(seg, ctxt->vcpu);
613 static unsigned long seg_override_base(struct x86_emulate_ctxt *ctxt,
614 struct x86_emulate_ops *ops,
615 struct decode_cache *c)
617 if (!c->has_seg_override)
620 return seg_base(ctxt, ops, c->seg_override);
623 static unsigned long es_base(struct x86_emulate_ctxt *ctxt,
624 struct x86_emulate_ops *ops)
626 return seg_base(ctxt, ops, VCPU_SREG_ES);
629 static unsigned long ss_base(struct x86_emulate_ctxt *ctxt,
630 struct x86_emulate_ops *ops)
632 return seg_base(ctxt, ops, VCPU_SREG_SS);
635 static void emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
636 u32 error, bool valid)
638 ctxt->exception = vec;
639 ctxt->error_code = error;
640 ctxt->error_code_valid = valid;
641 ctxt->restart = false;
644 static void emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
646 emulate_exception(ctxt, GP_VECTOR, err, true);
649 static void emulate_pf(struct x86_emulate_ctxt *ctxt, unsigned long addr,
653 emulate_exception(ctxt, PF_VECTOR, err, true);
656 static void emulate_ud(struct x86_emulate_ctxt *ctxt)
658 emulate_exception(ctxt, UD_VECTOR, 0, false);
661 static void emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
663 emulate_exception(ctxt, TS_VECTOR, err, true);
666 static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt,
667 struct x86_emulate_ops *ops,
668 unsigned long eip, u8 *dest)
670 struct fetch_cache *fc = &ctxt->decode.fetch;
674 if (eip == fc->end) {
675 cur_size = fc->end - fc->start;
676 size = min(15UL - cur_size, PAGE_SIZE - offset_in_page(eip));
677 rc = ops->fetch(ctxt->cs_base + eip, fc->data + cur_size,
678 size, ctxt->vcpu, NULL);
679 if (rc != X86EMUL_CONTINUE)
683 *dest = fc->data[eip - fc->start];
684 return X86EMUL_CONTINUE;
687 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
688 struct x86_emulate_ops *ops,
689 unsigned long eip, void *dest, unsigned size)
693 /* x86 instructions are limited to 15 bytes. */
694 if (eip + size - ctxt->eip > 15)
695 return X86EMUL_UNHANDLEABLE;
697 rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++);
698 if (rc != X86EMUL_CONTINUE)
701 return X86EMUL_CONTINUE;
705 * Given the 'reg' portion of a ModRM byte, and a register block, return a
706 * pointer into the block that addresses the relevant register.
707 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
709 static void *decode_register(u8 modrm_reg, unsigned long *regs,
714 p = ®s[modrm_reg];
715 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
716 p = (unsigned char *)®s[modrm_reg & 3] + 1;
720 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
721 struct x86_emulate_ops *ops,
723 u16 *size, unsigned long *address, int op_bytes)
730 rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2,
732 if (rc != X86EMUL_CONTINUE)
734 rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes,
739 static int test_cc(unsigned int condition, unsigned int flags)
743 switch ((condition & 15) >> 1) {
745 rc |= (flags & EFLG_OF);
747 case 1: /* b/c/nae */
748 rc |= (flags & EFLG_CF);
751 rc |= (flags & EFLG_ZF);
754 rc |= (flags & (EFLG_CF|EFLG_ZF));
757 rc |= (flags & EFLG_SF);
760 rc |= (flags & EFLG_PF);
763 rc |= (flags & EFLG_ZF);
766 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
770 /* Odd condition identifiers (lsb == 1) have inverted sense. */
771 return (!!rc ^ (condition & 1));
774 static void decode_register_operand(struct operand *op,
775 struct decode_cache *c,
778 unsigned reg = c->modrm_reg;
779 int highbyte_regs = c->rex_prefix == 0;
782 reg = (c->b & 7) | ((c->rex_prefix & 1) << 3);
784 if ((c->d & ByteOp) && !inhibit_bytereg) {
785 op->ptr = decode_register(reg, c->regs, highbyte_regs);
786 op->val = *(u8 *)op->ptr;
789 op->ptr = decode_register(reg, c->regs, 0);
790 op->bytes = c->op_bytes;
793 op->val = *(u16 *)op->ptr;
796 op->val = *(u32 *)op->ptr;
799 op->val = *(u64 *) op->ptr;
803 op->orig_val = op->val;
806 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
807 struct x86_emulate_ops *ops)
809 struct decode_cache *c = &ctxt->decode;
811 int index_reg = 0, base_reg = 0, scale;
812 int rc = X86EMUL_CONTINUE;
815 c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */
816 index_reg = (c->rex_prefix & 2) << 2; /* REX.X */
817 c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */
820 c->modrm = insn_fetch(u8, 1, c->eip);
821 c->modrm_mod |= (c->modrm & 0xc0) >> 6;
822 c->modrm_reg |= (c->modrm & 0x38) >> 3;
823 c->modrm_rm |= (c->modrm & 0x07);
827 if (c->modrm_mod == 3) {
828 c->modrm_ptr = decode_register(c->modrm_rm,
829 c->regs, c->d & ByteOp);
830 c->modrm_val = *(unsigned long *)c->modrm_ptr;
834 if (c->ad_bytes == 2) {
835 unsigned bx = c->regs[VCPU_REGS_RBX];
836 unsigned bp = c->regs[VCPU_REGS_RBP];
837 unsigned si = c->regs[VCPU_REGS_RSI];
838 unsigned di = c->regs[VCPU_REGS_RDI];
840 /* 16-bit ModR/M decode. */
841 switch (c->modrm_mod) {
843 if (c->modrm_rm == 6)
844 c->modrm_ea += insn_fetch(u16, 2, c->eip);
847 c->modrm_ea += insn_fetch(s8, 1, c->eip);
850 c->modrm_ea += insn_fetch(u16, 2, c->eip);
853 switch (c->modrm_rm) {
855 c->modrm_ea += bx + si;
858 c->modrm_ea += bx + di;
861 c->modrm_ea += bp + si;
864 c->modrm_ea += bp + di;
873 if (c->modrm_mod != 0)
880 if (c->modrm_rm == 2 || c->modrm_rm == 3 ||
881 (c->modrm_rm == 6 && c->modrm_mod != 0))
882 if (!c->has_seg_override)
883 set_seg_override(c, VCPU_SREG_SS);
884 c->modrm_ea = (u16)c->modrm_ea;
886 /* 32/64-bit ModR/M decode. */
887 if ((c->modrm_rm & 7) == 4) {
888 sib = insn_fetch(u8, 1, c->eip);
889 index_reg |= (sib >> 3) & 7;
893 if ((base_reg & 7) == 5 && c->modrm_mod == 0)
894 c->modrm_ea += insn_fetch(s32, 4, c->eip);
896 c->modrm_ea += c->regs[base_reg];
898 c->modrm_ea += c->regs[index_reg] << scale;
899 } else if ((c->modrm_rm & 7) == 5 && c->modrm_mod == 0) {
900 if (ctxt->mode == X86EMUL_MODE_PROT64)
903 c->modrm_ea += c->regs[c->modrm_rm];
904 switch (c->modrm_mod) {
906 if (c->modrm_rm == 5)
907 c->modrm_ea += insn_fetch(s32, 4, c->eip);
910 c->modrm_ea += insn_fetch(s8, 1, c->eip);
913 c->modrm_ea += insn_fetch(s32, 4, c->eip);
921 static int decode_abs(struct x86_emulate_ctxt *ctxt,
922 struct x86_emulate_ops *ops)
924 struct decode_cache *c = &ctxt->decode;
925 int rc = X86EMUL_CONTINUE;
927 switch (c->ad_bytes) {
929 c->modrm_ea = insn_fetch(u16, 2, c->eip);
932 c->modrm_ea = insn_fetch(u32, 4, c->eip);
935 c->modrm_ea = insn_fetch(u64, 8, c->eip);
943 x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
945 struct decode_cache *c = &ctxt->decode;
946 int rc = X86EMUL_CONTINUE;
947 int mode = ctxt->mode;
948 int def_op_bytes, def_ad_bytes, group, dual, goffset;
949 struct opcode opcode, *g_mod012, *g_mod3;
951 /* we cannot decode insn before we complete previous rep insn */
952 WARN_ON(ctxt->restart);
955 c->fetch.start = c->fetch.end = c->eip;
956 ctxt->cs_base = seg_base(ctxt, ops, VCPU_SREG_CS);
959 case X86EMUL_MODE_REAL:
960 case X86EMUL_MODE_VM86:
961 case X86EMUL_MODE_PROT16:
962 def_op_bytes = def_ad_bytes = 2;
964 case X86EMUL_MODE_PROT32:
965 def_op_bytes = def_ad_bytes = 4;
968 case X86EMUL_MODE_PROT64:
977 c->op_bytes = def_op_bytes;
978 c->ad_bytes = def_ad_bytes;
980 /* Legacy prefixes. */
982 switch (c->b = insn_fetch(u8, 1, c->eip)) {
983 case 0x66: /* operand-size override */
984 /* switch between 2/4 bytes */
985 c->op_bytes = def_op_bytes ^ 6;
987 case 0x67: /* address-size override */
988 if (mode == X86EMUL_MODE_PROT64)
989 /* switch between 4/8 bytes */
990 c->ad_bytes = def_ad_bytes ^ 12;
992 /* switch between 2/4 bytes */
993 c->ad_bytes = def_ad_bytes ^ 6;
995 case 0x26: /* ES override */
996 case 0x2e: /* CS override */
997 case 0x36: /* SS override */
998 case 0x3e: /* DS override */
999 set_seg_override(c, (c->b >> 3) & 3);
1001 case 0x64: /* FS override */
1002 case 0x65: /* GS override */
1003 set_seg_override(c, c->b & 7);
1005 case 0x40 ... 0x4f: /* REX */
1006 if (mode != X86EMUL_MODE_PROT64)
1008 c->rex_prefix = c->b;
1010 case 0xf0: /* LOCK */
1013 case 0xf2: /* REPNE/REPNZ */
1014 c->rep_prefix = REPNE_PREFIX;
1016 case 0xf3: /* REP/REPE/REPZ */
1017 c->rep_prefix = REPE_PREFIX;
1023 /* Any legacy prefix after a REX prefix nullifies its effect. */
1032 if (c->rex_prefix & 8)
1033 c->op_bytes = 8; /* REX.W */
1035 /* Opcode byte(s). */
1036 opcode = opcode_table[c->b];
1037 if (opcode.flags == 0) {
1038 /* Two-byte opcode? */
1041 c->b = insn_fetch(u8, 1, c->eip);
1042 opcode = twobyte_table[c->b];
1045 c->d = opcode.flags;
1048 group = c->d & GroupMask;
1049 dual = c->d & GroupDual;
1050 c->modrm = insn_fetch(u8, 1, c->eip);
1054 g_mod012 = g_mod3 = &group_table[group * 8];
1055 if (c->d & GroupDual)
1056 g_mod3 = &group2_table[group * 8];
1058 if (c->d & GroupDual) {
1059 g_mod012 = opcode.u.gdual->mod012;
1060 g_mod3 = opcode.u.gdual->mod3;
1062 g_mod012 = g_mod3 = opcode.u.group;
1065 c->d &= ~(Group | GroupDual | GroupMask);
1067 goffset = (c->modrm >> 3) & 7;
1069 if ((c->modrm >> 6) == 3)
1070 opcode = g_mod3[goffset];
1072 opcode = g_mod012[goffset];
1073 c->d |= opcode.flags;
1077 if (c->d == 0 || (c->d & Undefined)) {
1078 DPRINTF("Cannot emulate %02x\n", c->b);
1082 if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack))
1085 /* ModRM and SIB bytes. */
1087 rc = decode_modrm(ctxt, ops);
1088 else if (c->d & MemAbs)
1089 rc = decode_abs(ctxt, ops);
1090 if (rc != X86EMUL_CONTINUE)
1093 if (!c->has_seg_override)
1094 set_seg_override(c, VCPU_SREG_DS);
1096 if (!(!c->twobyte && c->b == 0x8d))
1097 c->modrm_ea += seg_override_base(ctxt, ops, c);
1099 if (c->ad_bytes != 8)
1100 c->modrm_ea = (u32)c->modrm_ea;
1102 if (c->rip_relative)
1103 c->modrm_ea += c->eip;
1106 * Decode and fetch the source operand: register, memory
1109 switch (c->d & SrcMask) {
1113 decode_register_operand(&c->src, c, 0);
1122 c->src.bytes = (c->d & ByteOp) ? 1 :
1124 /* Don't fetch the address for invlpg: it could be unmapped. */
1125 if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7)
1129 * For instructions with a ModR/M byte, switch to register
1130 * access if Mod = 3.
1132 if ((c->d & ModRM) && c->modrm_mod == 3) {
1133 c->src.type = OP_REG;
1134 c->src.val = c->modrm_val;
1135 c->src.ptr = c->modrm_ptr;
1138 c->src.type = OP_MEM;
1139 c->src.ptr = (unsigned long *)c->modrm_ea;
1144 c->src.type = OP_IMM;
1145 c->src.ptr = (unsigned long *)c->eip;
1146 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1147 if (c->src.bytes == 8)
1149 /* NB. Immediates are sign-extended as necessary. */
1150 switch (c->src.bytes) {
1152 c->src.val = insn_fetch(s8, 1, c->eip);
1155 c->src.val = insn_fetch(s16, 2, c->eip);
1158 c->src.val = insn_fetch(s32, 4, c->eip);
1161 if ((c->d & SrcMask) == SrcImmU) {
1162 switch (c->src.bytes) {
1167 c->src.val &= 0xffff;
1170 c->src.val &= 0xffffffff;
1177 c->src.type = OP_IMM;
1178 c->src.ptr = (unsigned long *)c->eip;
1180 if ((c->d & SrcMask) == SrcImmByte)
1181 c->src.val = insn_fetch(s8, 1, c->eip);
1183 c->src.val = insn_fetch(u8, 1, c->eip);
1186 c->src.type = OP_REG;
1187 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1188 c->src.ptr = &c->regs[VCPU_REGS_RAX];
1189 switch (c->src.bytes) {
1191 c->src.val = *(u8 *)c->src.ptr;
1194 c->src.val = *(u16 *)c->src.ptr;
1197 c->src.val = *(u32 *)c->src.ptr;
1200 c->src.val = *(u64 *)c->src.ptr;
1209 c->src.type = OP_MEM;
1210 c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1211 c->src.ptr = (unsigned long *)
1212 register_address(c, seg_override_base(ctxt, ops, c),
1213 c->regs[VCPU_REGS_RSI]);
1217 c->src.type = OP_IMM;
1218 c->src.ptr = (unsigned long *)c->eip;
1219 c->src.bytes = c->op_bytes + 2;
1220 insn_fetch_arr(c->src.valptr, c->src.bytes, c->eip);
1223 c->src.type = OP_MEM;
1224 c->src.ptr = (unsigned long *)c->modrm_ea;
1225 c->src.bytes = c->op_bytes + 2;
1230 * Decode and fetch the second source operand: register, memory
1233 switch (c->d & Src2Mask) {
1238 c->src2.val = c->regs[VCPU_REGS_RCX] & 0x8;
1241 c->src2.type = OP_IMM;
1242 c->src2.ptr = (unsigned long *)c->eip;
1244 c->src2.val = insn_fetch(u8, 1, c->eip);
1252 /* Decode and fetch the destination operand: register or memory. */
1253 switch (c->d & DstMask) {
1255 /* Special instructions do their own operand decoding. */
1258 decode_register_operand(&c->dst, c,
1259 c->twobyte && (c->b == 0xb6 || c->b == 0xb7));
1263 if ((c->d & ModRM) && c->modrm_mod == 3) {
1264 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1265 c->dst.type = OP_REG;
1266 c->dst.val = c->dst.orig_val = c->modrm_val;
1267 c->dst.ptr = c->modrm_ptr;
1270 c->dst.type = OP_MEM;
1271 c->dst.ptr = (unsigned long *)c->modrm_ea;
1272 if ((c->d & DstMask) == DstMem64)
1275 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1278 unsigned long mask = ~(c->dst.bytes * 8 - 1);
1280 c->dst.ptr = (void *)c->dst.ptr +
1281 (c->src.val & mask) / 8;
1285 c->dst.type = OP_REG;
1286 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1287 c->dst.ptr = &c->regs[VCPU_REGS_RAX];
1288 switch (c->dst.bytes) {
1290 c->dst.val = *(u8 *)c->dst.ptr;
1293 c->dst.val = *(u16 *)c->dst.ptr;
1296 c->dst.val = *(u32 *)c->dst.ptr;
1299 c->dst.val = *(u64 *)c->dst.ptr;
1302 c->dst.orig_val = c->dst.val;
1305 c->dst.type = OP_MEM;
1306 c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes;
1307 c->dst.ptr = (unsigned long *)
1308 register_address(c, es_base(ctxt, ops),
1309 c->regs[VCPU_REGS_RDI]);
1315 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
1318 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1319 struct x86_emulate_ops *ops,
1320 unsigned long addr, void *dest, unsigned size)
1323 struct read_cache *mc = &ctxt->decode.mem_read;
1327 int n = min(size, 8u);
1329 if (mc->pos < mc->end)
1332 rc = ops->read_emulated(addr, mc->data + mc->end, n, &err,
1334 if (rc == X86EMUL_PROPAGATE_FAULT)
1335 emulate_pf(ctxt, addr, err);
1336 if (rc != X86EMUL_CONTINUE)
1341 memcpy(dest, mc->data + mc->pos, n);
1346 return X86EMUL_CONTINUE;
1349 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1350 struct x86_emulate_ops *ops,
1351 unsigned int size, unsigned short port,
1354 struct read_cache *rc = &ctxt->decode.io_read;
1356 if (rc->pos == rc->end) { /* refill pio read ahead */
1357 struct decode_cache *c = &ctxt->decode;
1358 unsigned int in_page, n;
1359 unsigned int count = c->rep_prefix ?
1360 address_mask(c, c->regs[VCPU_REGS_RCX]) : 1;
1361 in_page = (ctxt->eflags & EFLG_DF) ?
1362 offset_in_page(c->regs[VCPU_REGS_RDI]) :
1363 PAGE_SIZE - offset_in_page(c->regs[VCPU_REGS_RDI]);
1364 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1368 rc->pos = rc->end = 0;
1369 if (!ops->pio_in_emulated(size, port, rc->data, n, ctxt->vcpu))
1374 memcpy(dest, rc->data + rc->pos, size);
1379 static u32 desc_limit_scaled(struct desc_struct *desc)
1381 u32 limit = get_desc_limit(desc);
1383 return desc->g ? (limit << 12) | 0xfff : limit;
1386 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1387 struct x86_emulate_ops *ops,
1388 u16 selector, struct desc_ptr *dt)
1390 if (selector & 1 << 2) {
1391 struct desc_struct desc;
1392 memset (dt, 0, sizeof *dt);
1393 if (!ops->get_cached_descriptor(&desc, VCPU_SREG_LDTR, ctxt->vcpu))
1396 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1397 dt->address = get_desc_base(&desc);
1399 ops->get_gdt(dt, ctxt->vcpu);
1402 /* allowed just for 8 bytes segments */
1403 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1404 struct x86_emulate_ops *ops,
1405 u16 selector, struct desc_struct *desc)
1408 u16 index = selector >> 3;
1413 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1415 if (dt.size < index * 8 + 7) {
1416 emulate_gp(ctxt, selector & 0xfffc);
1417 return X86EMUL_PROPAGATE_FAULT;
1419 addr = dt.address + index * 8;
1420 ret = ops->read_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1421 if (ret == X86EMUL_PROPAGATE_FAULT)
1422 emulate_pf(ctxt, addr, err);
1427 /* allowed just for 8 bytes segments */
1428 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1429 struct x86_emulate_ops *ops,
1430 u16 selector, struct desc_struct *desc)
1433 u16 index = selector >> 3;
1438 get_descriptor_table_ptr(ctxt, ops, selector, &dt);
1440 if (dt.size < index * 8 + 7) {
1441 emulate_gp(ctxt, selector & 0xfffc);
1442 return X86EMUL_PROPAGATE_FAULT;
1445 addr = dt.address + index * 8;
1446 ret = ops->write_std(addr, desc, sizeof *desc, ctxt->vcpu, &err);
1447 if (ret == X86EMUL_PROPAGATE_FAULT)
1448 emulate_pf(ctxt, addr, err);
1453 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1454 struct x86_emulate_ops *ops,
1455 u16 selector, int seg)
1457 struct desc_struct seg_desc;
1459 unsigned err_vec = GP_VECTOR;
1461 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1464 memset(&seg_desc, 0, sizeof seg_desc);
1466 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1467 || ctxt->mode == X86EMUL_MODE_REAL) {
1468 /* set real mode segment descriptor */
1469 set_desc_base(&seg_desc, selector << 4);
1470 set_desc_limit(&seg_desc, 0xffff);
1477 /* NULL selector is not valid for TR, CS and SS */
1478 if ((seg == VCPU_SREG_CS || seg == VCPU_SREG_SS || seg == VCPU_SREG_TR)
1482 /* TR should be in GDT only */
1483 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1486 if (null_selector) /* for NULL selector skip all following checks */
1489 ret = read_segment_descriptor(ctxt, ops, selector, &seg_desc);
1490 if (ret != X86EMUL_CONTINUE)
1493 err_code = selector & 0xfffc;
1494 err_vec = GP_VECTOR;
1496 /* can't load system descriptor into segment selecor */
1497 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1501 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1507 cpl = ops->cpl(ctxt->vcpu);
1512 * segment is not a writable data segment or segment
1513 * selector's RPL != CPL or segment selector's RPL != CPL
1515 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1519 if (!(seg_desc.type & 8))
1522 if (seg_desc.type & 4) {
1528 if (rpl > cpl || dpl != cpl)
1531 /* CS(RPL) <- CPL */
1532 selector = (selector & 0xfffc) | cpl;
1535 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1538 case VCPU_SREG_LDTR:
1539 if (seg_desc.s || seg_desc.type != 2)
1542 default: /* DS, ES, FS, or GS */
1544 * segment is not a data or readable code segment or
1545 * ((segment is a data or nonconforming code segment)
1546 * and (both RPL and CPL > DPL))
1548 if ((seg_desc.type & 0xa) == 0x8 ||
1549 (((seg_desc.type & 0xc) != 0xc) &&
1550 (rpl > dpl && cpl > dpl)))
1556 /* mark segment as accessed */
1558 ret = write_segment_descriptor(ctxt, ops, selector, &seg_desc);
1559 if (ret != X86EMUL_CONTINUE)
1563 ops->set_segment_selector(selector, seg, ctxt->vcpu);
1564 ops->set_cached_descriptor(&seg_desc, seg, ctxt->vcpu);
1565 return X86EMUL_CONTINUE;
1567 emulate_exception(ctxt, err_vec, err_code, true);
1568 return X86EMUL_PROPAGATE_FAULT;
1571 static inline int writeback(struct x86_emulate_ctxt *ctxt,
1572 struct x86_emulate_ops *ops)
1575 struct decode_cache *c = &ctxt->decode;
1578 switch (c->dst.type) {
1580 /* The 4-byte case *is* correct:
1581 * in 64-bit mode we zero-extend.
1583 switch (c->dst.bytes) {
1585 *(u8 *)c->dst.ptr = (u8)c->dst.val;
1588 *(u16 *)c->dst.ptr = (u16)c->dst.val;
1591 *c->dst.ptr = (u32)c->dst.val;
1592 break; /* 64b: zero-ext */
1594 *c->dst.ptr = c->dst.val;
1600 rc = ops->cmpxchg_emulated(
1601 (unsigned long)c->dst.ptr,
1608 rc = ops->write_emulated(
1609 (unsigned long)c->dst.ptr,
1614 if (rc == X86EMUL_PROPAGATE_FAULT)
1616 (unsigned long)c->dst.ptr, err);
1617 if (rc != X86EMUL_CONTINUE)
1626 return X86EMUL_CONTINUE;
1629 static inline void emulate_push(struct x86_emulate_ctxt *ctxt,
1630 struct x86_emulate_ops *ops)
1632 struct decode_cache *c = &ctxt->decode;
1634 c->dst.type = OP_MEM;
1635 c->dst.bytes = c->op_bytes;
1636 c->dst.val = c->src.val;
1637 register_address_increment(c, &c->regs[VCPU_REGS_RSP], -c->op_bytes);
1638 c->dst.ptr = (void *) register_address(c, ss_base(ctxt, ops),
1639 c->regs[VCPU_REGS_RSP]);
1642 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1643 struct x86_emulate_ops *ops,
1644 void *dest, int len)
1646 struct decode_cache *c = &ctxt->decode;
1649 rc = read_emulated(ctxt, ops, register_address(c, ss_base(ctxt, ops),
1650 c->regs[VCPU_REGS_RSP]),
1652 if (rc != X86EMUL_CONTINUE)
1655 register_address_increment(c, &c->regs[VCPU_REGS_RSP], len);
1659 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1660 struct x86_emulate_ops *ops,
1661 void *dest, int len)
1664 unsigned long val, change_mask;
1665 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1666 int cpl = ops->cpl(ctxt->vcpu);
1668 rc = emulate_pop(ctxt, ops, &val, len);
1669 if (rc != X86EMUL_CONTINUE)
1672 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1673 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1675 switch(ctxt->mode) {
1676 case X86EMUL_MODE_PROT64:
1677 case X86EMUL_MODE_PROT32:
1678 case X86EMUL_MODE_PROT16:
1680 change_mask |= EFLG_IOPL;
1682 change_mask |= EFLG_IF;
1684 case X86EMUL_MODE_VM86:
1686 emulate_gp(ctxt, 0);
1687 return X86EMUL_PROPAGATE_FAULT;
1689 change_mask |= EFLG_IF;
1691 default: /* real mode */
1692 change_mask |= (EFLG_IOPL | EFLG_IF);
1696 *(unsigned long *)dest =
1697 (ctxt->eflags & ~change_mask) | (val & change_mask);
1702 static void emulate_push_sreg(struct x86_emulate_ctxt *ctxt,
1703 struct x86_emulate_ops *ops, int seg)
1705 struct decode_cache *c = &ctxt->decode;
1707 c->src.val = ops->get_segment_selector(seg, ctxt->vcpu);
1709 emulate_push(ctxt, ops);
1712 static int emulate_pop_sreg(struct x86_emulate_ctxt *ctxt,
1713 struct x86_emulate_ops *ops, int seg)
1715 struct decode_cache *c = &ctxt->decode;
1716 unsigned long selector;
1719 rc = emulate_pop(ctxt, ops, &selector, c->op_bytes);
1720 if (rc != X86EMUL_CONTINUE)
1723 rc = load_segment_descriptor(ctxt, ops, (u16)selector, seg);
1727 static int emulate_pusha(struct x86_emulate_ctxt *ctxt,
1728 struct x86_emulate_ops *ops)
1730 struct decode_cache *c = &ctxt->decode;
1731 unsigned long old_esp = c->regs[VCPU_REGS_RSP];
1732 int rc = X86EMUL_CONTINUE;
1733 int reg = VCPU_REGS_RAX;
1735 while (reg <= VCPU_REGS_RDI) {
1736 (reg == VCPU_REGS_RSP) ?
1737 (c->src.val = old_esp) : (c->src.val = c->regs[reg]);
1739 emulate_push(ctxt, ops);
1741 rc = writeback(ctxt, ops);
1742 if (rc != X86EMUL_CONTINUE)
1748 /* Disable writeback. */
1749 c->dst.type = OP_NONE;
1754 static int emulate_popa(struct x86_emulate_ctxt *ctxt,
1755 struct x86_emulate_ops *ops)
1757 struct decode_cache *c = &ctxt->decode;
1758 int rc = X86EMUL_CONTINUE;
1759 int reg = VCPU_REGS_RDI;
1761 while (reg >= VCPU_REGS_RAX) {
1762 if (reg == VCPU_REGS_RSP) {
1763 register_address_increment(c, &c->regs[VCPU_REGS_RSP],
1768 rc = emulate_pop(ctxt, ops, &c->regs[reg], c->op_bytes);
1769 if (rc != X86EMUL_CONTINUE)
1776 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt,
1777 struct x86_emulate_ops *ops)
1779 struct decode_cache *c = &ctxt->decode;
1780 int rc = X86EMUL_CONTINUE;
1781 unsigned long temp_eip = 0;
1782 unsigned long temp_eflags = 0;
1783 unsigned long cs = 0;
1784 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1785 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1786 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1787 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1789 /* TODO: Add stack limit check */
1791 rc = emulate_pop(ctxt, ops, &temp_eip, c->op_bytes);
1793 if (rc != X86EMUL_CONTINUE)
1796 if (temp_eip & ~0xffff) {
1797 emulate_gp(ctxt, 0);
1798 return X86EMUL_PROPAGATE_FAULT;
1801 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1803 if (rc != X86EMUL_CONTINUE)
1806 rc = emulate_pop(ctxt, ops, &temp_eflags, c->op_bytes);
1808 if (rc != X86EMUL_CONTINUE)
1811 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1813 if (rc != X86EMUL_CONTINUE)
1819 if (c->op_bytes == 4)
1820 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1821 else if (c->op_bytes == 2) {
1822 ctxt->eflags &= ~0xffff;
1823 ctxt->eflags |= temp_eflags;
1826 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1827 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1832 static inline int emulate_iret(struct x86_emulate_ctxt *ctxt,
1833 struct x86_emulate_ops* ops)
1835 switch(ctxt->mode) {
1836 case X86EMUL_MODE_REAL:
1837 return emulate_iret_real(ctxt, ops);
1838 case X86EMUL_MODE_VM86:
1839 case X86EMUL_MODE_PROT16:
1840 case X86EMUL_MODE_PROT32:
1841 case X86EMUL_MODE_PROT64:
1843 /* iret from protected mode unimplemented yet */
1844 return X86EMUL_UNHANDLEABLE;
1848 static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt,
1849 struct x86_emulate_ops *ops)
1851 struct decode_cache *c = &ctxt->decode;
1853 return emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes);
1856 static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt)
1858 struct decode_cache *c = &ctxt->decode;
1859 switch (c->modrm_reg) {
1861 emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags);
1864 emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags);
1867 emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags);
1870 emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags);
1872 case 4: /* sal/shl */
1873 case 6: /* sal/shl */
1874 emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags);
1877 emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags);
1880 emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags);
1885 static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt,
1886 struct x86_emulate_ops *ops)
1888 struct decode_cache *c = &ctxt->decode;
1890 switch (c->modrm_reg) {
1891 case 0 ... 1: /* test */
1892 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
1895 c->dst.val = ~c->dst.val;
1898 emulate_1op("neg", c->dst, ctxt->eflags);
1906 static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt,
1907 struct x86_emulate_ops *ops)
1909 struct decode_cache *c = &ctxt->decode;
1911 switch (c->modrm_reg) {
1913 emulate_1op("inc", c->dst, ctxt->eflags);
1916 emulate_1op("dec", c->dst, ctxt->eflags);
1918 case 2: /* call near abs */ {
1921 c->eip = c->src.val;
1922 c->src.val = old_eip;
1923 emulate_push(ctxt, ops);
1926 case 4: /* jmp abs */
1927 c->eip = c->src.val;
1930 emulate_push(ctxt, ops);
1933 return X86EMUL_CONTINUE;
1936 static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt,
1937 struct x86_emulate_ops *ops)
1939 struct decode_cache *c = &ctxt->decode;
1940 u64 old = c->dst.orig_val64;
1942 if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) ||
1943 ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) {
1944 c->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
1945 c->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
1946 ctxt->eflags &= ~EFLG_ZF;
1948 c->dst.val64 = ((u64)c->regs[VCPU_REGS_RCX] << 32) |
1949 (u32) c->regs[VCPU_REGS_RBX];
1951 ctxt->eflags |= EFLG_ZF;
1953 return X86EMUL_CONTINUE;
1956 static int emulate_ret_far(struct x86_emulate_ctxt *ctxt,
1957 struct x86_emulate_ops *ops)
1959 struct decode_cache *c = &ctxt->decode;
1963 rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes);
1964 if (rc != X86EMUL_CONTINUE)
1966 if (c->op_bytes == 4)
1967 c->eip = (u32)c->eip;
1968 rc = emulate_pop(ctxt, ops, &cs, c->op_bytes);
1969 if (rc != X86EMUL_CONTINUE)
1971 rc = load_segment_descriptor(ctxt, ops, (u16)cs, VCPU_SREG_CS);
1976 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
1977 struct x86_emulate_ops *ops, struct desc_struct *cs,
1978 struct desc_struct *ss)
1980 memset(cs, 0, sizeof(struct desc_struct));
1981 ops->get_cached_descriptor(cs, VCPU_SREG_CS, ctxt->vcpu);
1982 memset(ss, 0, sizeof(struct desc_struct));
1984 cs->l = 0; /* will be adjusted later */
1985 set_desc_base(cs, 0); /* flat segment */
1986 cs->g = 1; /* 4kb granularity */
1987 set_desc_limit(cs, 0xfffff); /* 4GB limit */
1988 cs->type = 0x0b; /* Read, Execute, Accessed */
1990 cs->dpl = 0; /* will be adjusted later */
1994 set_desc_base(ss, 0); /* flat segment */
1995 set_desc_limit(ss, 0xfffff); /* 4GB limit */
1996 ss->g = 1; /* 4kb granularity */
1998 ss->type = 0x03; /* Read/Write, Accessed */
1999 ss->d = 1; /* 32bit stack segment */
2005 emulate_syscall(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2007 struct decode_cache *c = &ctxt->decode;
2008 struct desc_struct cs, ss;
2012 /* syscall is not available in real mode */
2013 if (ctxt->mode == X86EMUL_MODE_REAL ||
2014 ctxt->mode == X86EMUL_MODE_VM86) {
2016 return X86EMUL_PROPAGATE_FAULT;
2019 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2021 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2023 cs_sel = (u16)(msr_data & 0xfffc);
2024 ss_sel = (u16)(msr_data + 8);
2026 if (is_long_mode(ctxt->vcpu)) {
2030 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2031 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2032 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2033 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2035 c->regs[VCPU_REGS_RCX] = c->eip;
2036 if (is_long_mode(ctxt->vcpu)) {
2037 #ifdef CONFIG_X86_64
2038 c->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
2040 ops->get_msr(ctxt->vcpu,
2041 ctxt->mode == X86EMUL_MODE_PROT64 ?
2042 MSR_LSTAR : MSR_CSTAR, &msr_data);
2045 ops->get_msr(ctxt->vcpu, MSR_SYSCALL_MASK, &msr_data);
2046 ctxt->eflags &= ~(msr_data | EFLG_RF);
2050 ops->get_msr(ctxt->vcpu, MSR_STAR, &msr_data);
2051 c->eip = (u32)msr_data;
2053 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2056 return X86EMUL_CONTINUE;
2060 emulate_sysenter(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2062 struct decode_cache *c = &ctxt->decode;
2063 struct desc_struct cs, ss;
2067 /* inject #GP if in real mode */
2068 if (ctxt->mode == X86EMUL_MODE_REAL) {
2069 emulate_gp(ctxt, 0);
2070 return X86EMUL_PROPAGATE_FAULT;
2073 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2074 * Therefore, we inject an #UD.
2076 if (ctxt->mode == X86EMUL_MODE_PROT64) {
2078 return X86EMUL_PROPAGATE_FAULT;
2081 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2083 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2084 switch (ctxt->mode) {
2085 case X86EMUL_MODE_PROT32:
2086 if ((msr_data & 0xfffc) == 0x0) {
2087 emulate_gp(ctxt, 0);
2088 return X86EMUL_PROPAGATE_FAULT;
2091 case X86EMUL_MODE_PROT64:
2092 if (msr_data == 0x0) {
2093 emulate_gp(ctxt, 0);
2094 return X86EMUL_PROPAGATE_FAULT;
2099 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2100 cs_sel = (u16)msr_data;
2101 cs_sel &= ~SELECTOR_RPL_MASK;
2102 ss_sel = cs_sel + 8;
2103 ss_sel &= ~SELECTOR_RPL_MASK;
2104 if (ctxt->mode == X86EMUL_MODE_PROT64
2105 || is_long_mode(ctxt->vcpu)) {
2110 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2111 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2112 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2113 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2115 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_EIP, &msr_data);
2118 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_ESP, &msr_data);
2119 c->regs[VCPU_REGS_RSP] = msr_data;
2121 return X86EMUL_CONTINUE;
2125 emulate_sysexit(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2127 struct decode_cache *c = &ctxt->decode;
2128 struct desc_struct cs, ss;
2133 /* inject #GP if in real mode or Virtual 8086 mode */
2134 if (ctxt->mode == X86EMUL_MODE_REAL ||
2135 ctxt->mode == X86EMUL_MODE_VM86) {
2136 emulate_gp(ctxt, 0);
2137 return X86EMUL_PROPAGATE_FAULT;
2140 setup_syscalls_segments(ctxt, ops, &cs, &ss);
2142 if ((c->rex_prefix & 0x8) != 0x0)
2143 usermode = X86EMUL_MODE_PROT64;
2145 usermode = X86EMUL_MODE_PROT32;
2149 ops->get_msr(ctxt->vcpu, MSR_IA32_SYSENTER_CS, &msr_data);
2151 case X86EMUL_MODE_PROT32:
2152 cs_sel = (u16)(msr_data + 16);
2153 if ((msr_data & 0xfffc) == 0x0) {
2154 emulate_gp(ctxt, 0);
2155 return X86EMUL_PROPAGATE_FAULT;
2157 ss_sel = (u16)(msr_data + 24);
2159 case X86EMUL_MODE_PROT64:
2160 cs_sel = (u16)(msr_data + 32);
2161 if (msr_data == 0x0) {
2162 emulate_gp(ctxt, 0);
2163 return X86EMUL_PROPAGATE_FAULT;
2165 ss_sel = cs_sel + 8;
2170 cs_sel |= SELECTOR_RPL_MASK;
2171 ss_sel |= SELECTOR_RPL_MASK;
2173 ops->set_cached_descriptor(&cs, VCPU_SREG_CS, ctxt->vcpu);
2174 ops->set_segment_selector(cs_sel, VCPU_SREG_CS, ctxt->vcpu);
2175 ops->set_cached_descriptor(&ss, VCPU_SREG_SS, ctxt->vcpu);
2176 ops->set_segment_selector(ss_sel, VCPU_SREG_SS, ctxt->vcpu);
2178 c->eip = c->regs[VCPU_REGS_RDX];
2179 c->regs[VCPU_REGS_RSP] = c->regs[VCPU_REGS_RCX];
2181 return X86EMUL_CONTINUE;
2184 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt,
2185 struct x86_emulate_ops *ops)
2188 if (ctxt->mode == X86EMUL_MODE_REAL)
2190 if (ctxt->mode == X86EMUL_MODE_VM86)
2192 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2193 return ops->cpl(ctxt->vcpu) > iopl;
2196 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2197 struct x86_emulate_ops *ops,
2200 struct desc_struct tr_seg;
2203 u8 perm, bit_idx = port & 0x7;
2204 unsigned mask = (1 << len) - 1;
2206 ops->get_cached_descriptor(&tr_seg, VCPU_SREG_TR, ctxt->vcpu);
2209 if (desc_limit_scaled(&tr_seg) < 103)
2211 r = ops->read_std(get_desc_base(&tr_seg) + 102, &io_bitmap_ptr, 2,
2213 if (r != X86EMUL_CONTINUE)
2215 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2217 r = ops->read_std(get_desc_base(&tr_seg) + io_bitmap_ptr + port/8,
2218 &perm, 1, ctxt->vcpu, NULL);
2219 if (r != X86EMUL_CONTINUE)
2221 if ((perm >> bit_idx) & mask)
2226 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2227 struct x86_emulate_ops *ops,
2230 if (emulator_bad_iopl(ctxt, ops))
2231 if (!emulator_io_port_access_allowed(ctxt, ops, port, len))
2236 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2237 struct x86_emulate_ops *ops,
2238 struct tss_segment_16 *tss)
2240 struct decode_cache *c = &ctxt->decode;
2243 tss->flag = ctxt->eflags;
2244 tss->ax = c->regs[VCPU_REGS_RAX];
2245 tss->cx = c->regs[VCPU_REGS_RCX];
2246 tss->dx = c->regs[VCPU_REGS_RDX];
2247 tss->bx = c->regs[VCPU_REGS_RBX];
2248 tss->sp = c->regs[VCPU_REGS_RSP];
2249 tss->bp = c->regs[VCPU_REGS_RBP];
2250 tss->si = c->regs[VCPU_REGS_RSI];
2251 tss->di = c->regs[VCPU_REGS_RDI];
2253 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2254 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2255 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2256 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2257 tss->ldt = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2260 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2261 struct x86_emulate_ops *ops,
2262 struct tss_segment_16 *tss)
2264 struct decode_cache *c = &ctxt->decode;
2268 ctxt->eflags = tss->flag | 2;
2269 c->regs[VCPU_REGS_RAX] = tss->ax;
2270 c->regs[VCPU_REGS_RCX] = tss->cx;
2271 c->regs[VCPU_REGS_RDX] = tss->dx;
2272 c->regs[VCPU_REGS_RBX] = tss->bx;
2273 c->regs[VCPU_REGS_RSP] = tss->sp;
2274 c->regs[VCPU_REGS_RBP] = tss->bp;
2275 c->regs[VCPU_REGS_RSI] = tss->si;
2276 c->regs[VCPU_REGS_RDI] = tss->di;
2279 * SDM says that segment selectors are loaded before segment
2282 ops->set_segment_selector(tss->ldt, VCPU_SREG_LDTR, ctxt->vcpu);
2283 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2284 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2285 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2286 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2289 * Now load segment descriptors. If fault happenes at this stage
2290 * it is handled in a context of new task
2292 ret = load_segment_descriptor(ctxt, ops, tss->ldt, VCPU_SREG_LDTR);
2293 if (ret != X86EMUL_CONTINUE)
2295 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2296 if (ret != X86EMUL_CONTINUE)
2298 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2299 if (ret != X86EMUL_CONTINUE)
2301 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2302 if (ret != X86EMUL_CONTINUE)
2304 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2305 if (ret != X86EMUL_CONTINUE)
2308 return X86EMUL_CONTINUE;
2311 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2312 struct x86_emulate_ops *ops,
2313 u16 tss_selector, u16 old_tss_sel,
2314 ulong old_tss_base, struct desc_struct *new_desc)
2316 struct tss_segment_16 tss_seg;
2318 u32 err, new_tss_base = get_desc_base(new_desc);
2320 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2322 if (ret == X86EMUL_PROPAGATE_FAULT) {
2323 /* FIXME: need to provide precise fault address */
2324 emulate_pf(ctxt, old_tss_base, err);
2328 save_state_to_tss16(ctxt, ops, &tss_seg);
2330 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2332 if (ret == X86EMUL_PROPAGATE_FAULT) {
2333 /* FIXME: need to provide precise fault address */
2334 emulate_pf(ctxt, old_tss_base, err);
2338 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2340 if (ret == X86EMUL_PROPAGATE_FAULT) {
2341 /* FIXME: need to provide precise fault address */
2342 emulate_pf(ctxt, new_tss_base, err);
2346 if (old_tss_sel != 0xffff) {
2347 tss_seg.prev_task_link = old_tss_sel;
2349 ret = ops->write_std(new_tss_base,
2350 &tss_seg.prev_task_link,
2351 sizeof tss_seg.prev_task_link,
2353 if (ret == X86EMUL_PROPAGATE_FAULT) {
2354 /* FIXME: need to provide precise fault address */
2355 emulate_pf(ctxt, new_tss_base, err);
2360 return load_state_from_tss16(ctxt, ops, &tss_seg);
2363 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2364 struct x86_emulate_ops *ops,
2365 struct tss_segment_32 *tss)
2367 struct decode_cache *c = &ctxt->decode;
2369 tss->cr3 = ops->get_cr(3, ctxt->vcpu);
2371 tss->eflags = ctxt->eflags;
2372 tss->eax = c->regs[VCPU_REGS_RAX];
2373 tss->ecx = c->regs[VCPU_REGS_RCX];
2374 tss->edx = c->regs[VCPU_REGS_RDX];
2375 tss->ebx = c->regs[VCPU_REGS_RBX];
2376 tss->esp = c->regs[VCPU_REGS_RSP];
2377 tss->ebp = c->regs[VCPU_REGS_RBP];
2378 tss->esi = c->regs[VCPU_REGS_RSI];
2379 tss->edi = c->regs[VCPU_REGS_RDI];
2381 tss->es = ops->get_segment_selector(VCPU_SREG_ES, ctxt->vcpu);
2382 tss->cs = ops->get_segment_selector(VCPU_SREG_CS, ctxt->vcpu);
2383 tss->ss = ops->get_segment_selector(VCPU_SREG_SS, ctxt->vcpu);
2384 tss->ds = ops->get_segment_selector(VCPU_SREG_DS, ctxt->vcpu);
2385 tss->fs = ops->get_segment_selector(VCPU_SREG_FS, ctxt->vcpu);
2386 tss->gs = ops->get_segment_selector(VCPU_SREG_GS, ctxt->vcpu);
2387 tss->ldt_selector = ops->get_segment_selector(VCPU_SREG_LDTR, ctxt->vcpu);
2390 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2391 struct x86_emulate_ops *ops,
2392 struct tss_segment_32 *tss)
2394 struct decode_cache *c = &ctxt->decode;
2397 if (ops->set_cr(3, tss->cr3, ctxt->vcpu)) {
2398 emulate_gp(ctxt, 0);
2399 return X86EMUL_PROPAGATE_FAULT;
2402 ctxt->eflags = tss->eflags | 2;
2403 c->regs[VCPU_REGS_RAX] = tss->eax;
2404 c->regs[VCPU_REGS_RCX] = tss->ecx;
2405 c->regs[VCPU_REGS_RDX] = tss->edx;
2406 c->regs[VCPU_REGS_RBX] = tss->ebx;
2407 c->regs[VCPU_REGS_RSP] = tss->esp;
2408 c->regs[VCPU_REGS_RBP] = tss->ebp;
2409 c->regs[VCPU_REGS_RSI] = tss->esi;
2410 c->regs[VCPU_REGS_RDI] = tss->edi;
2413 * SDM says that segment selectors are loaded before segment
2416 ops->set_segment_selector(tss->ldt_selector, VCPU_SREG_LDTR, ctxt->vcpu);
2417 ops->set_segment_selector(tss->es, VCPU_SREG_ES, ctxt->vcpu);
2418 ops->set_segment_selector(tss->cs, VCPU_SREG_CS, ctxt->vcpu);
2419 ops->set_segment_selector(tss->ss, VCPU_SREG_SS, ctxt->vcpu);
2420 ops->set_segment_selector(tss->ds, VCPU_SREG_DS, ctxt->vcpu);
2421 ops->set_segment_selector(tss->fs, VCPU_SREG_FS, ctxt->vcpu);
2422 ops->set_segment_selector(tss->gs, VCPU_SREG_GS, ctxt->vcpu);
2425 * Now load segment descriptors. If fault happenes at this stage
2426 * it is handled in a context of new task
2428 ret = load_segment_descriptor(ctxt, ops, tss->ldt_selector, VCPU_SREG_LDTR);
2429 if (ret != X86EMUL_CONTINUE)
2431 ret = load_segment_descriptor(ctxt, ops, tss->es, VCPU_SREG_ES);
2432 if (ret != X86EMUL_CONTINUE)
2434 ret = load_segment_descriptor(ctxt, ops, tss->cs, VCPU_SREG_CS);
2435 if (ret != X86EMUL_CONTINUE)
2437 ret = load_segment_descriptor(ctxt, ops, tss->ss, VCPU_SREG_SS);
2438 if (ret != X86EMUL_CONTINUE)
2440 ret = load_segment_descriptor(ctxt, ops, tss->ds, VCPU_SREG_DS);
2441 if (ret != X86EMUL_CONTINUE)
2443 ret = load_segment_descriptor(ctxt, ops, tss->fs, VCPU_SREG_FS);
2444 if (ret != X86EMUL_CONTINUE)
2446 ret = load_segment_descriptor(ctxt, ops, tss->gs, VCPU_SREG_GS);
2447 if (ret != X86EMUL_CONTINUE)
2450 return X86EMUL_CONTINUE;
2453 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2454 struct x86_emulate_ops *ops,
2455 u16 tss_selector, u16 old_tss_sel,
2456 ulong old_tss_base, struct desc_struct *new_desc)
2458 struct tss_segment_32 tss_seg;
2460 u32 err, new_tss_base = get_desc_base(new_desc);
2462 ret = ops->read_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2464 if (ret == X86EMUL_PROPAGATE_FAULT) {
2465 /* FIXME: need to provide precise fault address */
2466 emulate_pf(ctxt, old_tss_base, err);
2470 save_state_to_tss32(ctxt, ops, &tss_seg);
2472 ret = ops->write_std(old_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2474 if (ret == X86EMUL_PROPAGATE_FAULT) {
2475 /* FIXME: need to provide precise fault address */
2476 emulate_pf(ctxt, old_tss_base, err);
2480 ret = ops->read_std(new_tss_base, &tss_seg, sizeof tss_seg, ctxt->vcpu,
2482 if (ret == X86EMUL_PROPAGATE_FAULT) {
2483 /* FIXME: need to provide precise fault address */
2484 emulate_pf(ctxt, new_tss_base, err);
2488 if (old_tss_sel != 0xffff) {
2489 tss_seg.prev_task_link = old_tss_sel;
2491 ret = ops->write_std(new_tss_base,
2492 &tss_seg.prev_task_link,
2493 sizeof tss_seg.prev_task_link,
2495 if (ret == X86EMUL_PROPAGATE_FAULT) {
2496 /* FIXME: need to provide precise fault address */
2497 emulate_pf(ctxt, new_tss_base, err);
2502 return load_state_from_tss32(ctxt, ops, &tss_seg);
2505 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2506 struct x86_emulate_ops *ops,
2507 u16 tss_selector, int reason,
2508 bool has_error_code, u32 error_code)
2510 struct desc_struct curr_tss_desc, next_tss_desc;
2512 u16 old_tss_sel = ops->get_segment_selector(VCPU_SREG_TR, ctxt->vcpu);
2513 ulong old_tss_base =
2514 ops->get_cached_segment_base(VCPU_SREG_TR, ctxt->vcpu);
2517 /* FIXME: old_tss_base == ~0 ? */
2519 ret = read_segment_descriptor(ctxt, ops, tss_selector, &next_tss_desc);
2520 if (ret != X86EMUL_CONTINUE)
2522 ret = read_segment_descriptor(ctxt, ops, old_tss_sel, &curr_tss_desc);
2523 if (ret != X86EMUL_CONTINUE)
2526 /* FIXME: check that next_tss_desc is tss */
2528 if (reason != TASK_SWITCH_IRET) {
2529 if ((tss_selector & 3) > next_tss_desc.dpl ||
2530 ops->cpl(ctxt->vcpu) > next_tss_desc.dpl) {
2531 emulate_gp(ctxt, 0);
2532 return X86EMUL_PROPAGATE_FAULT;
2536 desc_limit = desc_limit_scaled(&next_tss_desc);
2537 if (!next_tss_desc.p ||
2538 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2539 desc_limit < 0x2b)) {
2540 emulate_ts(ctxt, tss_selector & 0xfffc);
2541 return X86EMUL_PROPAGATE_FAULT;
2544 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2545 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2546 write_segment_descriptor(ctxt, ops, old_tss_sel,
2550 if (reason == TASK_SWITCH_IRET)
2551 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2553 /* set back link to prev task only if NT bit is set in eflags
2554 note that old_tss_sel is not used afetr this point */
2555 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2556 old_tss_sel = 0xffff;
2558 if (next_tss_desc.type & 8)
2559 ret = task_switch_32(ctxt, ops, tss_selector, old_tss_sel,
2560 old_tss_base, &next_tss_desc);
2562 ret = task_switch_16(ctxt, ops, tss_selector, old_tss_sel,
2563 old_tss_base, &next_tss_desc);
2564 if (ret != X86EMUL_CONTINUE)
2567 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2568 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2570 if (reason != TASK_SWITCH_IRET) {
2571 next_tss_desc.type |= (1 << 1); /* set busy flag */
2572 write_segment_descriptor(ctxt, ops, tss_selector,
2576 ops->set_cr(0, ops->get_cr(0, ctxt->vcpu) | X86_CR0_TS, ctxt->vcpu);
2577 ops->set_cached_descriptor(&next_tss_desc, VCPU_SREG_TR, ctxt->vcpu);
2578 ops->set_segment_selector(tss_selector, VCPU_SREG_TR, ctxt->vcpu);
2580 if (has_error_code) {
2581 struct decode_cache *c = &ctxt->decode;
2583 c->op_bytes = c->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2585 c->src.val = (unsigned long) error_code;
2586 emulate_push(ctxt, ops);
2592 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2593 struct x86_emulate_ops *ops,
2594 u16 tss_selector, int reason,
2595 bool has_error_code, u32 error_code)
2597 struct decode_cache *c = &ctxt->decode;
2601 c->dst.type = OP_NONE;
2603 rc = emulator_do_task_switch(ctxt, ops, tss_selector, reason,
2604 has_error_code, error_code);
2606 if (rc == X86EMUL_CONTINUE) {
2607 rc = writeback(ctxt, ops);
2608 if (rc == X86EMUL_CONTINUE)
2612 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
2615 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned long base,
2616 int reg, struct operand *op)
2618 struct decode_cache *c = &ctxt->decode;
2619 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
2621 register_address_increment(c, &c->regs[reg], df * op->bytes);
2622 op->ptr = (unsigned long *)register_address(c, base, c->regs[reg]);
2626 x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops)
2629 struct decode_cache *c = &ctxt->decode;
2630 int rc = X86EMUL_CONTINUE;
2631 int saved_dst_type = c->dst.type;
2633 ctxt->decode.mem_read.pos = 0;
2635 if (ctxt->mode == X86EMUL_MODE_PROT64 && (c->d & No64)) {
2640 /* LOCK prefix is allowed only with some instructions */
2641 if (c->lock_prefix && (!(c->d & Lock) || c->dst.type != OP_MEM)) {
2646 /* Privileged instruction can be executed only in CPL=0 */
2647 if ((c->d & Priv) && ops->cpl(ctxt->vcpu)) {
2648 emulate_gp(ctxt, 0);
2652 if (c->rep_prefix && (c->d & String)) {
2653 ctxt->restart = true;
2654 /* All REP prefixes have the same first termination condition */
2655 if (address_mask(c, c->regs[VCPU_REGS_RCX]) == 0) {
2657 ctxt->restart = false;
2661 /* The second termination condition only applies for REPE
2662 * and REPNE. Test if the repeat string operation prefix is
2663 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
2664 * corresponding termination condition according to:
2665 * - if REPE/REPZ and ZF = 0 then done
2666 * - if REPNE/REPNZ and ZF = 1 then done
2668 if ((c->b == 0xa6) || (c->b == 0xa7) ||
2669 (c->b == 0xae) || (c->b == 0xaf)) {
2670 if ((c->rep_prefix == REPE_PREFIX) &&
2671 ((ctxt->eflags & EFLG_ZF) == 0))
2673 if ((c->rep_prefix == REPNE_PREFIX) &&
2674 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))
2680 if (c->src.type == OP_MEM) {
2681 rc = read_emulated(ctxt, ops, (unsigned long)c->src.ptr,
2682 c->src.valptr, c->src.bytes);
2683 if (rc != X86EMUL_CONTINUE)
2685 c->src.orig_val64 = c->src.val64;
2688 if (c->src2.type == OP_MEM) {
2689 rc = read_emulated(ctxt, ops, (unsigned long)c->src2.ptr,
2690 &c->src2.val, c->src2.bytes);
2691 if (rc != X86EMUL_CONTINUE)
2695 if ((c->d & DstMask) == ImplicitOps)
2699 if ((c->dst.type == OP_MEM) && !(c->d & Mov)) {
2700 /* optimisation - avoid slow emulated read if Mov */
2701 rc = read_emulated(ctxt, ops, (unsigned long)c->dst.ptr,
2702 &c->dst.val, c->dst.bytes);
2703 if (rc != X86EMUL_CONTINUE)
2706 c->dst.orig_val = c->dst.val;
2716 emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags);
2718 case 0x06: /* push es */
2719 emulate_push_sreg(ctxt, ops, VCPU_SREG_ES);
2721 case 0x07: /* pop es */
2722 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_ES);
2723 if (rc != X86EMUL_CONTINUE)
2728 emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags);
2730 case 0x0e: /* push cs */
2731 emulate_push_sreg(ctxt, ops, VCPU_SREG_CS);
2735 emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags);
2737 case 0x16: /* push ss */
2738 emulate_push_sreg(ctxt, ops, VCPU_SREG_SS);
2740 case 0x17: /* pop ss */
2741 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_SS);
2742 if (rc != X86EMUL_CONTINUE)
2747 emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags);
2749 case 0x1e: /* push ds */
2750 emulate_push_sreg(ctxt, ops, VCPU_SREG_DS);
2752 case 0x1f: /* pop ds */
2753 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_DS);
2754 if (rc != X86EMUL_CONTINUE)
2759 emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags);
2763 emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags);
2767 emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags);
2771 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
2773 case 0x40 ... 0x47: /* inc r16/r32 */
2774 emulate_1op("inc", c->dst, ctxt->eflags);
2776 case 0x48 ... 0x4f: /* dec r16/r32 */
2777 emulate_1op("dec", c->dst, ctxt->eflags);
2779 case 0x50 ... 0x57: /* push reg */
2780 emulate_push(ctxt, ops);
2782 case 0x58 ... 0x5f: /* pop reg */
2784 rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes);
2785 if (rc != X86EMUL_CONTINUE)
2788 case 0x60: /* pusha */
2789 rc = emulate_pusha(ctxt, ops);
2790 if (rc != X86EMUL_CONTINUE)
2793 case 0x61: /* popa */
2794 rc = emulate_popa(ctxt, ops);
2795 if (rc != X86EMUL_CONTINUE)
2798 case 0x63: /* movsxd */
2799 if (ctxt->mode != X86EMUL_MODE_PROT64)
2800 goto cannot_emulate;
2801 c->dst.val = (s32) c->src.val;
2803 case 0x68: /* push imm */
2804 case 0x6a: /* push imm8 */
2805 emulate_push(ctxt, ops);
2807 case 0x6c: /* insb */
2808 case 0x6d: /* insw/insd */
2809 c->dst.bytes = min(c->dst.bytes, 4u);
2810 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2812 emulate_gp(ctxt, 0);
2815 if (!pio_in_emulated(ctxt, ops, c->dst.bytes,
2816 c->regs[VCPU_REGS_RDX], &c->dst.val))
2817 goto done; /* IO is needed, skip writeback */
2819 case 0x6e: /* outsb */
2820 case 0x6f: /* outsw/outsd */
2821 c->src.bytes = min(c->src.bytes, 4u);
2822 if (!emulator_io_permited(ctxt, ops, c->regs[VCPU_REGS_RDX],
2824 emulate_gp(ctxt, 0);
2827 ops->pio_out_emulated(c->src.bytes, c->regs[VCPU_REGS_RDX],
2828 &c->src.val, 1, ctxt->vcpu);
2830 c->dst.type = OP_NONE; /* nothing to writeback */
2832 case 0x70 ... 0x7f: /* jcc (short) */
2833 if (test_cc(c->b, ctxt->eflags))
2834 jmp_rel(c, c->src.val);
2836 case 0x80 ... 0x83: /* Grp1 */
2837 switch (c->modrm_reg) {
2858 emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags);
2860 case 0x86 ... 0x87: /* xchg */
2862 /* Write back the register source. */
2863 switch (c->dst.bytes) {
2865 *(u8 *) c->src.ptr = (u8) c->dst.val;
2868 *(u16 *) c->src.ptr = (u16) c->dst.val;
2871 *c->src.ptr = (u32) c->dst.val;
2872 break; /* 64b reg: zero-extend */
2874 *c->src.ptr = c->dst.val;
2878 * Write back the memory destination with implicit LOCK
2881 c->dst.val = c->src.val;
2884 case 0x88 ... 0x8b: /* mov */
2886 case 0x8c: /* mov r/m, sreg */
2887 if (c->modrm_reg > VCPU_SREG_GS) {
2891 c->dst.val = ops->get_segment_selector(c->modrm_reg, ctxt->vcpu);
2893 case 0x8d: /* lea r16/r32, m */
2894 c->dst.val = c->modrm_ea;
2896 case 0x8e: { /* mov seg, r/m16 */
2901 if (c->modrm_reg == VCPU_SREG_CS ||
2902 c->modrm_reg > VCPU_SREG_GS) {
2907 if (c->modrm_reg == VCPU_SREG_SS)
2908 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2910 rc = load_segment_descriptor(ctxt, ops, sel, c->modrm_reg);
2912 c->dst.type = OP_NONE; /* Disable writeback. */
2915 case 0x8f: /* pop (sole member of Grp1a) */
2916 rc = emulate_grp1a(ctxt, ops);
2917 if (rc != X86EMUL_CONTINUE)
2920 case 0x90: /* nop / xchg r8,rax */
2921 if (c->dst.ptr == (unsigned long *)&c->regs[VCPU_REGS_RAX]) {
2922 c->dst.type = OP_NONE; /* nop */
2925 case 0x91 ... 0x97: /* xchg reg,rax */
2926 c->src.type = OP_REG;
2927 c->src.bytes = c->op_bytes;
2928 c->src.ptr = (unsigned long *) &c->regs[VCPU_REGS_RAX];
2929 c->src.val = *(c->src.ptr);
2931 case 0x9c: /* pushf */
2932 c->src.val = (unsigned long) ctxt->eflags;
2933 emulate_push(ctxt, ops);
2935 case 0x9d: /* popf */
2936 c->dst.type = OP_REG;
2937 c->dst.ptr = (unsigned long *) &ctxt->eflags;
2938 c->dst.bytes = c->op_bytes;
2939 rc = emulate_popf(ctxt, ops, &c->dst.val, c->op_bytes);
2940 if (rc != X86EMUL_CONTINUE)
2943 case 0xa0 ... 0xa3: /* mov */
2944 case 0xa4 ... 0xa5: /* movs */
2946 case 0xa6 ... 0xa7: /* cmps */
2947 c->dst.type = OP_NONE; /* Disable writeback. */
2948 DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr);
2950 case 0xa8 ... 0xa9: /* test ax, imm */
2952 case 0xaa ... 0xab: /* stos */
2953 c->dst.val = c->regs[VCPU_REGS_RAX];
2955 case 0xac ... 0xad: /* lods */
2957 case 0xae ... 0xaf: /* scas */
2958 DPRINTF("Urk! I don't handle SCAS.\n");
2959 goto cannot_emulate;
2960 case 0xb0 ... 0xbf: /* mov r, imm */
2965 case 0xc3: /* ret */
2966 c->dst.type = OP_REG;
2967 c->dst.ptr = &c->eip;
2968 c->dst.bytes = c->op_bytes;
2969 goto pop_instruction;
2970 case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */
2972 c->dst.val = c->src.val;
2974 case 0xcb: /* ret far */
2975 rc = emulate_ret_far(ctxt, ops);
2976 if (rc != X86EMUL_CONTINUE)
2979 case 0xcf: /* iret */
2980 rc = emulate_iret(ctxt, ops);
2982 if (rc != X86EMUL_CONTINUE)
2985 case 0xd0 ... 0xd1: /* Grp2 */
2989 case 0xd2 ... 0xd3: /* Grp2 */
2990 c->src.val = c->regs[VCPU_REGS_RCX];
2993 case 0xe4: /* inb */
2996 case 0xe6: /* outb */
2997 case 0xe7: /* out */
2999 case 0xe8: /* call (near) */ {
3000 long int rel = c->src.val;
3001 c->src.val = (unsigned long) c->eip;
3003 emulate_push(ctxt, ops);
3006 case 0xe9: /* jmp rel */
3008 case 0xea: { /* jmp far */
3011 memcpy(&sel, c->src.valptr + c->op_bytes, 2);
3013 if (load_segment_descriptor(ctxt, ops, sel, VCPU_SREG_CS))
3017 memcpy(&c->eip, c->src.valptr, c->op_bytes);
3021 jmp: /* jmp rel short */
3022 jmp_rel(c, c->src.val);
3023 c->dst.type = OP_NONE; /* Disable writeback. */
3025 case 0xec: /* in al,dx */
3026 case 0xed: /* in (e/r)ax,dx */
3027 c->src.val = c->regs[VCPU_REGS_RDX];
3029 c->dst.bytes = min(c->dst.bytes, 4u);
3030 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3031 emulate_gp(ctxt, 0);
3034 if (!pio_in_emulated(ctxt, ops, c->dst.bytes, c->src.val,
3036 goto done; /* IO is needed */
3038 case 0xee: /* out dx,al */
3039 case 0xef: /* out dx,(e/r)ax */
3040 c->src.val = c->regs[VCPU_REGS_RDX];
3042 c->dst.bytes = min(c->dst.bytes, 4u);
3043 if (!emulator_io_permited(ctxt, ops, c->src.val, c->dst.bytes)) {
3044 emulate_gp(ctxt, 0);
3047 ops->pio_out_emulated(c->dst.bytes, c->src.val, &c->dst.val, 1,
3049 c->dst.type = OP_NONE; /* Disable writeback. */
3051 case 0xf4: /* hlt */
3052 ctxt->vcpu->arch.halt_request = 1;
3054 case 0xf5: /* cmc */
3055 /* complement carry flag from eflags reg */
3056 ctxt->eflags ^= EFLG_CF;
3057 c->dst.type = OP_NONE; /* Disable writeback. */
3059 case 0xf6 ... 0xf7: /* Grp3 */
3060 if (!emulate_grp3(ctxt, ops))
3061 goto cannot_emulate;
3063 case 0xf8: /* clc */
3064 ctxt->eflags &= ~EFLG_CF;
3065 c->dst.type = OP_NONE; /* Disable writeback. */
3067 case 0xfa: /* cli */
3068 if (emulator_bad_iopl(ctxt, ops)) {
3069 emulate_gp(ctxt, 0);
3072 ctxt->eflags &= ~X86_EFLAGS_IF;
3073 c->dst.type = OP_NONE; /* Disable writeback. */
3076 case 0xfb: /* sti */
3077 if (emulator_bad_iopl(ctxt, ops)) {
3078 emulate_gp(ctxt, 0);
3081 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3082 ctxt->eflags |= X86_EFLAGS_IF;
3083 c->dst.type = OP_NONE; /* Disable writeback. */
3086 case 0xfc: /* cld */
3087 ctxt->eflags &= ~EFLG_DF;
3088 c->dst.type = OP_NONE; /* Disable writeback. */
3090 case 0xfd: /* std */
3091 ctxt->eflags |= EFLG_DF;
3092 c->dst.type = OP_NONE; /* Disable writeback. */
3094 case 0xfe: /* Grp4 */
3096 rc = emulate_grp45(ctxt, ops);
3097 if (rc != X86EMUL_CONTINUE)
3100 case 0xff: /* Grp5 */
3101 if (c->modrm_reg == 5)
3105 goto cannot_emulate;
3109 rc = writeback(ctxt, ops);
3110 if (rc != X86EMUL_CONTINUE)
3114 * restore dst type in case the decoding will be reused
3115 * (happens for string instruction )
3117 c->dst.type = saved_dst_type;
3119 if ((c->d & SrcMask) == SrcSI)
3120 string_addr_inc(ctxt, seg_override_base(ctxt, ops, c),
3121 VCPU_REGS_RSI, &c->src);
3123 if ((c->d & DstMask) == DstDI)
3124 string_addr_inc(ctxt, es_base(ctxt, ops), VCPU_REGS_RDI,
3127 if (c->rep_prefix && (c->d & String)) {
3128 struct read_cache *rc = &ctxt->decode.io_read;
3129 register_address_increment(c, &c->regs[VCPU_REGS_RCX], -1);
3131 * Re-enter guest when pio read ahead buffer is empty or,
3132 * if it is not used, after each 1024 iteration.
3134 if ((rc->end == 0 && !(c->regs[VCPU_REGS_RCX] & 0x3ff)) ||
3135 (rc->end != 0 && rc->end == rc->pos))
3136 ctxt->restart = false;
3139 * reset read cache here in case string instruction is restared
3142 ctxt->decode.mem_read.end = 0;
3146 return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0;
3150 case 0x01: /* lgdt, lidt, lmsw */
3151 switch (c->modrm_reg) {
3153 unsigned long address;
3155 case 0: /* vmcall */
3156 if (c->modrm_mod != 3 || c->modrm_rm != 1)
3157 goto cannot_emulate;
3159 rc = kvm_fix_hypercall(ctxt->vcpu);
3160 if (rc != X86EMUL_CONTINUE)
3163 /* Let the processor re-execute the fixed hypercall */
3165 /* Disable writeback. */
3166 c->dst.type = OP_NONE;
3169 rc = read_descriptor(ctxt, ops, c->src.ptr,
3170 &size, &address, c->op_bytes);
3171 if (rc != X86EMUL_CONTINUE)
3173 realmode_lgdt(ctxt->vcpu, size, address);
3174 /* Disable writeback. */
3175 c->dst.type = OP_NONE;
3177 case 3: /* lidt/vmmcall */
3178 if (c->modrm_mod == 3) {
3179 switch (c->modrm_rm) {
3181 rc = kvm_fix_hypercall(ctxt->vcpu);
3182 if (rc != X86EMUL_CONTINUE)
3186 goto cannot_emulate;
3189 rc = read_descriptor(ctxt, ops, c->src.ptr,
3192 if (rc != X86EMUL_CONTINUE)
3194 realmode_lidt(ctxt->vcpu, size, address);
3196 /* Disable writeback. */
3197 c->dst.type = OP_NONE;
3201 c->dst.val = ops->get_cr(0, ctxt->vcpu);
3204 ops->set_cr(0, (ops->get_cr(0, ctxt->vcpu) & ~0x0ful) |
3205 (c->src.val & 0x0f), ctxt->vcpu);
3206 c->dst.type = OP_NONE;
3208 case 5: /* not defined */
3212 emulate_invlpg(ctxt->vcpu, c->modrm_ea);
3213 /* Disable writeback. */
3214 c->dst.type = OP_NONE;
3217 goto cannot_emulate;
3220 case 0x05: /* syscall */
3221 rc = emulate_syscall(ctxt, ops);
3222 if (rc != X86EMUL_CONTINUE)
3228 emulate_clts(ctxt->vcpu);
3229 c->dst.type = OP_NONE;
3231 case 0x09: /* wbinvd */
3232 kvm_emulate_wbinvd(ctxt->vcpu);
3233 c->dst.type = OP_NONE;
3235 case 0x08: /* invd */
3236 case 0x0d: /* GrpP (prefetch) */
3237 case 0x18: /* Grp16 (prefetch/nop) */
3238 c->dst.type = OP_NONE;
3240 case 0x20: /* mov cr, reg */
3241 switch (c->modrm_reg) {
3248 c->regs[c->modrm_rm] = ops->get_cr(c->modrm_reg, ctxt->vcpu);
3249 c->dst.type = OP_NONE; /* no writeback */
3251 case 0x21: /* mov from dr to reg */
3252 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3253 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3257 ops->get_dr(c->modrm_reg, &c->regs[c->modrm_rm], ctxt->vcpu);
3258 c->dst.type = OP_NONE; /* no writeback */
3260 case 0x22: /* mov reg, cr */
3261 if (ops->set_cr(c->modrm_reg, c->modrm_val, ctxt->vcpu)) {
3262 emulate_gp(ctxt, 0);
3265 c->dst.type = OP_NONE;
3267 case 0x23: /* mov from reg to dr */
3268 if ((ops->get_cr(4, ctxt->vcpu) & X86_CR4_DE) &&
3269 (c->modrm_reg == 4 || c->modrm_reg == 5)) {
3274 if (ops->set_dr(c->modrm_reg, c->regs[c->modrm_rm] &
3275 ((ctxt->mode == X86EMUL_MODE_PROT64) ?
3276 ~0ULL : ~0U), ctxt->vcpu) < 0) {
3277 /* #UD condition is already handled by the code above */
3278 emulate_gp(ctxt, 0);
3282 c->dst.type = OP_NONE; /* no writeback */
3286 msr_data = (u32)c->regs[VCPU_REGS_RAX]
3287 | ((u64)c->regs[VCPU_REGS_RDX] << 32);
3288 if (ops->set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data)) {
3289 emulate_gp(ctxt, 0);
3292 rc = X86EMUL_CONTINUE;
3293 c->dst.type = OP_NONE;
3297 if (ops->get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data)) {
3298 emulate_gp(ctxt, 0);
3301 c->regs[VCPU_REGS_RAX] = (u32)msr_data;
3302 c->regs[VCPU_REGS_RDX] = msr_data >> 32;
3304 rc = X86EMUL_CONTINUE;
3305 c->dst.type = OP_NONE;
3307 case 0x34: /* sysenter */
3308 rc = emulate_sysenter(ctxt, ops);
3309 if (rc != X86EMUL_CONTINUE)
3314 case 0x35: /* sysexit */
3315 rc = emulate_sysexit(ctxt, ops);
3316 if (rc != X86EMUL_CONTINUE)
3321 case 0x40 ... 0x4f: /* cmov */
3322 c->dst.val = c->dst.orig_val = c->src.val;
3323 if (!test_cc(c->b, ctxt->eflags))
3324 c->dst.type = OP_NONE; /* no writeback */
3326 case 0x80 ... 0x8f: /* jnz rel, etc*/
3327 if (test_cc(c->b, ctxt->eflags))
3328 jmp_rel(c, c->src.val);
3329 c->dst.type = OP_NONE;
3331 case 0xa0: /* push fs */
3332 emulate_push_sreg(ctxt, ops, VCPU_SREG_FS);
3334 case 0xa1: /* pop fs */
3335 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_FS);
3336 if (rc != X86EMUL_CONTINUE)
3341 c->dst.type = OP_NONE;
3342 /* only subword offset */
3343 c->src.val &= (c->dst.bytes << 3) - 1;
3344 emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags);
3346 case 0xa4: /* shld imm8, r, r/m */
3347 case 0xa5: /* shld cl, r, r/m */
3348 emulate_2op_cl("shld", c->src2, c->src, c->dst, ctxt->eflags);
3350 case 0xa8: /* push gs */
3351 emulate_push_sreg(ctxt, ops, VCPU_SREG_GS);
3353 case 0xa9: /* pop gs */
3354 rc = emulate_pop_sreg(ctxt, ops, VCPU_SREG_GS);
3355 if (rc != X86EMUL_CONTINUE)
3360 /* only subword offset */
3361 c->src.val &= (c->dst.bytes << 3) - 1;
3362 emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags);
3364 case 0xac: /* shrd imm8, r, r/m */
3365 case 0xad: /* shrd cl, r, r/m */
3366 emulate_2op_cl("shrd", c->src2, c->src, c->dst, ctxt->eflags);
3368 case 0xae: /* clflush */
3370 case 0xb0 ... 0xb1: /* cmpxchg */
3372 * Save real source value, then compare EAX against
3375 c->src.orig_val = c->src.val;
3376 c->src.val = c->regs[VCPU_REGS_RAX];
3377 emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags);
3378 if (ctxt->eflags & EFLG_ZF) {
3379 /* Success: write back to memory. */
3380 c->dst.val = c->src.orig_val;
3382 /* Failure: write the value we saw to EAX. */
3383 c->dst.type = OP_REG;
3384 c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX];
3389 /* only subword offset */
3390 c->src.val &= (c->dst.bytes << 3) - 1;
3391 emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags);
3393 case 0xb6 ... 0xb7: /* movzx */
3394 c->dst.bytes = c->op_bytes;
3395 c->dst.val = (c->d & ByteOp) ? (u8) c->src.val
3398 case 0xba: /* Grp8 */
3399 switch (c->modrm_reg & 3) {
3412 /* only subword offset */
3413 c->src.val &= (c->dst.bytes << 3) - 1;
3414 emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags);
3416 case 0xbe ... 0xbf: /* movsx */
3417 c->dst.bytes = c->op_bytes;
3418 c->dst.val = (c->d & ByteOp) ? (s8) c->src.val :
3421 case 0xc3: /* movnti */
3422 c->dst.bytes = c->op_bytes;
3423 c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val :
3426 case 0xc7: /* Grp9 (cmpxchg8b) */
3427 rc = emulate_grp9(ctxt, ops);
3428 if (rc != X86EMUL_CONTINUE)
3432 goto cannot_emulate;
3437 DPRINTF("Cannot emulate %02x\n", c->b);