Lines Matching +full:op +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
27 #include <asm/nospec-branch.h>
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
223 struct opcode op[8]; member
248 unsigned long dirty = ctxt->regs_dirty; in writeback_registers()
252 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); in writeback_registers()
257 ctxt->regs_dirty = 0; in invalidate_registers()
258 ctxt->regs_valid = 0; in invalidate_registers()
307 ".size " name ", .-" name "\n\t"
312 #define __FOP_START(op, align) \ argument
313 extern void em_##op(struct fastop *fake); \
315 ".global em_" #op " \n\t" \
317 "em_" #op ":\n\t"
319 #define FOP_START(op) __FOP_START(op, FASTOP_SIZE) argument
331 #define FOP1E(op, dst) \ argument
332 __FOP_FUNC(#op "_" #dst) \
333 "10: " #op " %" #dst " \n\t" \
334 __FOP_RET(#op "_" #dst)
336 #define FOP1EEX(op, dst) \ argument
337 FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
339 #define FASTOP1(op) \ argument
340 FOP_START(op) \
341 FOP1E(op##b, al) \
342 FOP1E(op##w, ax) \
343 FOP1E(op##l, eax) \
344 ON64(FOP1E(op##q, rax)) \
347 /* 1-operand, using src2 (for MUL/DIV r/m) */
348 #define FASTOP1SRC2(op, name) \ argument
350 FOP1E(op, cl) \
351 FOP1E(op, cx) \
352 FOP1E(op, ecx) \
353 ON64(FOP1E(op, rcx)) \
356 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
357 #define FASTOP1SRC2EX(op, name) \ argument
359 FOP1EEX(op, cl) \
360 FOP1EEX(op, cx) \
361 FOP1EEX(op, ecx) \
362 ON64(FOP1EEX(op, rcx)) \
365 #define FOP2E(op, dst, src) \ argument
366 __FOP_FUNC(#op "_" #dst "_" #src) \
367 #op " %" #src ", %" #dst " \n\t" \
368 __FOP_RET(#op "_" #dst "_" #src)
370 #define FASTOP2(op) \ argument
371 FOP_START(op) \
372 FOP2E(op##b, al, dl) \
373 FOP2E(op##w, ax, dx) \
374 FOP2E(op##l, eax, edx) \
375 ON64(FOP2E(op##q, rax, rdx)) \
379 #define FASTOP2W(op) \ argument
380 FOP_START(op) \
382 FOP2E(op##w, ax, dx) \
383 FOP2E(op##l, eax, edx) \
384 ON64(FOP2E(op##q, rax, rdx)) \
388 #define FASTOP2CL(op) \ argument
389 FOP_START(op) \
390 FOP2E(op##b, al, cl) \
391 FOP2E(op##w, ax, cl) \
392 FOP2E(op##l, eax, cl) \
393 ON64(FOP2E(op##q, rax, cl)) \
397 #define FASTOP2R(op, name) \ argument
399 FOP2E(op##b, dl, al) \
400 FOP2E(op##w, dx, ax) \
401 FOP2E(op##l, edx, eax) \
402 ON64(FOP2E(op##q, rdx, rax)) \
405 #define FOP3E(op, dst, src, src2) \ argument
406 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
407 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
408 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \ argument
412 FOP_START(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \ argument
421 FOP_FUNC(op) \
422 #op " %al \n\t" \
423 FOP_RET(op)
472 .rep_prefix = ctxt->rep_prefix, in emulator_check_intercept()
473 .modrm_mod = ctxt->modrm_mod, in emulator_check_intercept()
474 .modrm_reg = ctxt->modrm_reg, in emulator_check_intercept()
475 .modrm_rm = ctxt->modrm_rm, in emulator_check_intercept()
476 .src_val = ctxt->src.val64, in emulator_check_intercept()
477 .dst_val = ctxt->dst.val64, in emulator_check_intercept()
478 .src_bytes = ctxt->src.bytes, in emulator_check_intercept()
479 .dst_bytes = ctxt->dst.bytes, in emulator_check_intercept()
480 .src_type = ctxt->src.type, in emulator_check_intercept()
481 .dst_type = ctxt->dst.type, in emulator_check_intercept()
482 .ad_bytes = ctxt->ad_bytes, in emulator_check_intercept()
483 .rip = ctxt->eip, in emulator_check_intercept()
484 .next_rip = ctxt->_eip, in emulator_check_intercept()
487 return ctxt->ops->intercept(ctxt, &info, stage); in emulator_check_intercept()
497 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ in assign_register()
507 break; /* 64b: zero-extend */ in assign_register()
516 return (1UL << (ctxt->ad_bytes << 3)) - 1; in ad_mask()
524 if (ctxt->mode == X86EMUL_MODE_PROT64) in stack_mask()
526 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS); in stack_mask()
535 /* Access/update address held in a register, based on addressing mode. */
539 if (ctxt->ad_bytes == sizeof(unsigned long)) in address_mask()
561 assign_register(preg, *preg + inc, ctxt->ad_bytes); in register_address_increment()
573 return desc->g ? (limit << 12) | 0xfff : limit; in desc_limit_scaled()
578 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS) in seg_base()
581 return ctxt->ops->get_cached_segment_base(ctxt, seg); in seg_base()
590 ctxt->exception.vector = vec; in emulate_exception()
591 ctxt->exception.error_code = error; in emulate_exception()
592 ctxt->exception.error_code_valid = valid; in emulate_exception()
636 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg); in get_segment_selector()
647 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg); in set_segment_selector()
648 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg); in set_segment_selector()
653 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; in ctxt_virt_addr_bits()
660 return !ctxt->ops->is_canonical_addr(ctxt, la, flags); in emul_is_noncanonical_address()
674 u64 alignment = ctxt->d & AlignMask; in insn_alignment()
694 enum x86emul_mode mode, ulong *linear, in __linearize() argument
706 switch (mode) { in __linearize()
708 *linear = la = ctxt->ops->get_untagged_addr(ctxt, la, flags); in __linearize()
713 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la); in __linearize()
719 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL, in __linearize()
723 /* code segment in protected mode or read-only data segment */ in __linearize()
724 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) || !(desc.type & 2)) && in __linearize()
732 /* expand-down segment */ in __linearize()
742 *max_size = (u64)lim + 1 - addr.ea; in __linearize()
748 if (la & (insn_alignment(ctxt, size) - 1)) in __linearize()
764 return __linearize(ctxt, addr, &max_size, size, ctxt->mode, linear, in linearize()
776 if (ctxt->op_bytes != sizeof(unsigned long)) in assign_eip()
777 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); in assign_eip()
778 rc = __linearize(ctxt, addr, &max_size, 1, ctxt->mode, &linear, in assign_eip()
781 ctxt->_eip = addr.ea; in assign_eip()
792 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in emulator_recalc_and_set_mode()
794 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) { in emulator_recalc_and_set_mode()
795 /* Real mode. cpu must not have long mode active */ in emulator_recalc_and_set_mode()
798 ctxt->mode = X86EMUL_MODE_REAL; in emulator_recalc_and_set_mode()
802 if (ctxt->eflags & X86_EFLAGS_VM) { in emulator_recalc_and_set_mode()
803 /* Protected/VM86 mode. cpu must not have long mode active */ in emulator_recalc_and_set_mode()
806 ctxt->mode = X86EMUL_MODE_VM86; in emulator_recalc_and_set_mode()
810 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS)) in emulator_recalc_and_set_mode()
815 /* Proper long mode */ in emulator_recalc_and_set_mode()
816 ctxt->mode = X86EMUL_MODE_PROT64; in emulator_recalc_and_set_mode()
818 /* 32 bit compatibility mode*/ in emulator_recalc_and_set_mode()
819 ctxt->mode = X86EMUL_MODE_PROT32; in emulator_recalc_and_set_mode()
821 ctxt->mode = X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
824 /* Legacy 32 bit / 16 bit mode */ in emulator_recalc_and_set_mode()
825 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
848 return assign_eip_near(ctxt, ctxt->_eip + rel); in jmp_rel()
854 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_read_system()
861 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_write_system()
875 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_read_std()
889 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_write_std()
901 int cur_size = ctxt->fetch.end - ctxt->fetch.data; in __do_insn_fetch_bytes()
903 .ea = ctxt->eip + cur_size }; in __do_insn_fetch_bytes()
915 rc = __linearize(ctxt, addr, &max_size, 0, ctxt->mode, &linear, in __do_insn_fetch_bytes()
921 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear)); in __do_insn_fetch_bytes()
927 * still, we must have hit the 15-byte boundary. in __do_insn_fetch_bytes()
932 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end, in __do_insn_fetch_bytes()
933 size, &ctxt->exception); in __do_insn_fetch_bytes()
936 ctxt->fetch.end += size; in __do_insn_fetch_bytes()
943 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr; in do_insn_fetch_bytes()
946 return __do_insn_fetch_bytes(ctxt, size - done_size); in do_insn_fetch_bytes()
958 ctxt->_eip += sizeof(_type); \
959 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
960 ctxt->fetch.ptr += sizeof(_type); \
969 ctxt->_eip += (_size); \
970 memcpy(_arr, ctxt->fetch.ptr, _size); \
971 ctxt->fetch.ptr += (_size); \
983 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop; in decode_register()
1056 if (ctxt->src.val == 0) in em_bsf_c()
1057 ctxt->dst.type = OP_NONE; in em_bsf_c()
1064 if (ctxt->src.val == 0) in em_bsr_c()
1065 ctxt->dst.type = OP_NONE; in em_bsr_c()
1080 static void fetch_register_operand(struct operand *op) in fetch_register_operand() argument
1082 switch (op->bytes) { in fetch_register_operand()
1084 op->val = *(u8 *)op->addr.reg; in fetch_register_operand()
1087 op->val = *(u16 *)op->addr.reg; in fetch_register_operand()
1090 op->val = *(u32 *)op->addr.reg; in fetch_register_operand()
1093 op->val = *(u64 *)op->addr.reg; in fetch_register_operand()
1100 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fninit()
1113 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstcw()
1120 ctxt->dst.val = fcw; in em_fnstcw()
1129 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstsw()
1136 ctxt->dst.val = fsw; in em_fnstsw()
1142 struct operand *op) in decode_register_operand() argument
1146 if (ctxt->d & ModRM) in decode_register_operand()
1147 reg = ctxt->modrm_reg; in decode_register_operand()
1149 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3); in decode_register_operand()
1151 if (ctxt->d & Sse) { in decode_register_operand()
1152 op->type = OP_XMM; in decode_register_operand()
1153 op->bytes = 16; in decode_register_operand()
1154 op->addr.xmm = reg; in decode_register_operand()
1155 kvm_read_sse_reg(reg, &op->vec_val); in decode_register_operand()
1158 if (ctxt->d & Mmx) { in decode_register_operand()
1160 op->type = OP_MM; in decode_register_operand()
1161 op->bytes = 8; in decode_register_operand()
1162 op->addr.mm = reg; in decode_register_operand()
1166 op->type = OP_REG; in decode_register_operand()
1167 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_register_operand()
1168 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp); in decode_register_operand()
1170 fetch_register_operand(op); in decode_register_operand()
1171 op->orig_val = op->val; in decode_register_operand()
1177 ctxt->modrm_seg = VCPU_SREG_SS; in adjust_modrm_seg()
1181 struct operand *op) in decode_modrm() argument
1188 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */ in decode_modrm()
1189 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */ in decode_modrm()
1190 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */ in decode_modrm()
1192 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6; in decode_modrm()
1193 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3; in decode_modrm()
1194 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07); in decode_modrm()
1195 ctxt->modrm_seg = VCPU_SREG_DS; in decode_modrm()
1197 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) { in decode_modrm()
1198 op->type = OP_REG; in decode_modrm()
1199 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_modrm()
1200 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, in decode_modrm()
1201 ctxt->d & ByteOp); in decode_modrm()
1202 if (ctxt->d & Sse) { in decode_modrm()
1203 op->type = OP_XMM; in decode_modrm()
1204 op->bytes = 16; in decode_modrm()
1205 op->addr.xmm = ctxt->modrm_rm; in decode_modrm()
1206 kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val); in decode_modrm()
1209 if (ctxt->d & Mmx) { in decode_modrm()
1210 op->type = OP_MM; in decode_modrm()
1211 op->bytes = 8; in decode_modrm()
1212 op->addr.mm = ctxt->modrm_rm & 7; in decode_modrm()
1215 fetch_register_operand(op); in decode_modrm()
1219 op->type = OP_MEM; in decode_modrm()
1221 if (ctxt->ad_bytes == 2) { in decode_modrm()
1227 /* 16-bit ModR/M decode. */ in decode_modrm()
1228 switch (ctxt->modrm_mod) { in decode_modrm()
1230 if (ctxt->modrm_rm == 6) in decode_modrm()
1240 switch (ctxt->modrm_rm) { in decode_modrm()
1260 if (ctxt->modrm_mod != 0) in decode_modrm()
1267 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 || in decode_modrm()
1268 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0)) in decode_modrm()
1269 ctxt->modrm_seg = VCPU_SREG_SS; in decode_modrm()
1272 /* 32/64-bit ModR/M decode. */ in decode_modrm()
1273 if ((ctxt->modrm_rm & 7) == 4) { in decode_modrm()
1279 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) in decode_modrm()
1285 if ((ctxt->d & IncSP) && in decode_modrm()
1287 modrm_ea += ctxt->op_bytes; in decode_modrm()
1291 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { in decode_modrm()
1293 if (ctxt->mode == X86EMUL_MODE_PROT64) in decode_modrm()
1294 ctxt->rip_relative = 1; in decode_modrm()
1296 base_reg = ctxt->modrm_rm; in decode_modrm()
1300 switch (ctxt->modrm_mod) { in decode_modrm()
1309 op->addr.mem.ea = modrm_ea; in decode_modrm()
1310 if (ctxt->ad_bytes != 8) in decode_modrm()
1311 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea; in decode_modrm()
1318 struct operand *op) in decode_abs() argument
1322 op->type = OP_MEM; in decode_abs()
1323 switch (ctxt->ad_bytes) { in decode_abs()
1325 op->addr.mem.ea = insn_fetch(u16, ctxt); in decode_abs()
1328 op->addr.mem.ea = insn_fetch(u32, ctxt); in decode_abs()
1331 op->addr.mem.ea = insn_fetch(u64, ctxt); in decode_abs()
1342 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) { in fetch_bit_operand()
1343 mask = ~((long)ctxt->dst.bytes * 8 - 1); in fetch_bit_operand()
1345 if (ctxt->src.bytes == 2) in fetch_bit_operand()
1346 sv = (s16)ctxt->src.val & (s16)mask; in fetch_bit_operand()
1347 else if (ctxt->src.bytes == 4) in fetch_bit_operand()
1348 sv = (s32)ctxt->src.val & (s32)mask; in fetch_bit_operand()
1350 sv = (s64)ctxt->src.val & (s64)mask; in fetch_bit_operand()
1352 ctxt->dst.addr.mem.ea = address_mask(ctxt, in fetch_bit_operand()
1353 ctxt->dst.addr.mem.ea + (sv >> 3)); in fetch_bit_operand()
1357 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1; in fetch_bit_operand()
1364 struct read_cache *mc = &ctxt->mem_read; in read_emulated()
1366 if (mc->pos < mc->end) in read_emulated()
1369 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt)) in read_emulated()
1372 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size, in read_emulated()
1373 &ctxt->exception); in read_emulated()
1377 mc->end += size; in read_emulated()
1380 memcpy(dest, mc->data + mc->pos, size); in read_emulated()
1381 mc->pos += size; in read_emulated()
1410 return ctxt->ops->write_emulated(ctxt, linear, data, size, in segmented_write()
1411 &ctxt->exception); in segmented_write()
1425 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data, in segmented_cmpxchg()
1426 size, &ctxt->exception); in segmented_cmpxchg()
1433 struct read_cache *rc = &ctxt->io_read; in pio_in_emulated()
1435 if (rc->pos == rc->end) { /* refill pio read ahead */ in pio_in_emulated()
1437 unsigned int count = ctxt->rep_prefix ? in pio_in_emulated()
1439 in_page = (ctxt->eflags & X86_EFLAGS_DF) ? in pio_in_emulated()
1441 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); in pio_in_emulated()
1442 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); in pio_in_emulated()
1445 rc->pos = rc->end = 0; in pio_in_emulated()
1446 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n)) in pio_in_emulated()
1448 rc->end = n * size; in pio_in_emulated()
1451 if (ctxt->rep_prefix && (ctxt->d & String) && in pio_in_emulated()
1452 !(ctxt->eflags & X86_EFLAGS_DF)) { in pio_in_emulated()
1453 ctxt->dst.data = rc->data + rc->pos; in pio_in_emulated()
1454 ctxt->dst.type = OP_MEM_STR; in pio_in_emulated()
1455 ctxt->dst.count = (rc->end - rc->pos) / size; in pio_in_emulated()
1456 rc->pos = rc->end; in pio_in_emulated()
1458 memcpy(dest, rc->data + rc->pos, size); in pio_in_emulated()
1459 rc->pos += size; in pio_in_emulated()
1470 ctxt->ops->get_idt(ctxt, &dt); in read_interrupt_descriptor()
1482 const struct x86_emulate_ops *ops = ctxt->ops; in get_descriptor_table_ptr()
1490 if (!ops->get_segment(ctxt, &sel, &desc, &base3, in get_descriptor_table_ptr()
1494 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */ in get_descriptor_table_ptr()
1495 dt->address = get_desc_base(&desc) | ((u64)base3 << 32); in get_descriptor_table_ptr()
1497 ops->get_gdt(ctxt, dt); in get_descriptor_table_ptr()
1518 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in get_descriptor_ptr()
1520 addr &= (u32)-1; in get_descriptor_ptr()
1565 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ in __load_segment_descriptor()
1573 if (ctxt->mode == X86EMUL_MODE_REAL) { in __load_segment_descriptor()
1574 /* set real mode segment descriptor (keep limit etc. for in __load_segment_descriptor()
1575 * unreal mode) */ in __load_segment_descriptor()
1576 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg); in __load_segment_descriptor()
1579 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) { in __load_segment_descriptor()
1596 /* NULL selector is not valid for TR, CS and (except for long mode) SS */ in __load_segment_descriptor()
1602 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) in __load_segment_descriptor()
1606 * ctxt->ops->set_segment expects the CPL to be in in __load_segment_descriptor()
1607 * SS.DPL, so fake an expand-up 32-bit data segment. in __load_segment_descriptor()
1650 * Mode exceptions and IRET (handled above). In all other in __load_segment_descriptor()
1663 /* Outer-privilege level return is not implemented */ in __load_segment_descriptor()
1688 /* in long-mode d/b must be clear if l is set */ in __load_segment_descriptor()
1692 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in __load_segment_descriptor()
1697 /* CS(RPL) <- CPL */ in __load_segment_descriptor()
1735 } else if (ctxt->mode == X86EMUL_MODE_PROT64) { in __load_segment_descriptor()
1748 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc, in __load_segment_descriptor()
1749 sizeof(seg_desc), &ctxt->exception); in __load_segment_descriptor()
1754 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg); in __load_segment_descriptor()
1765 u8 cpl = ctxt->ops->cpl(ctxt); in load_segment_descriptor()
1778 ctxt->mode == X86EMUL_MODE_PROT64) in load_segment_descriptor()
1785 static void write_register_operand(struct operand *op) in write_register_operand() argument
1787 return assign_register(op->addr.reg, op->val, op->bytes); in write_register_operand()
1790 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) in writeback() argument
1792 switch (op->type) { in writeback()
1794 write_register_operand(op); in writeback()
1797 if (ctxt->lock_prefix) in writeback()
1799 op->addr.mem, in writeback()
1800 &op->orig_val, in writeback()
1801 &op->val, in writeback()
1802 op->bytes); in writeback()
1805 op->addr.mem, in writeback()
1806 &op->val, in writeback()
1807 op->bytes); in writeback()
1810 op->addr.mem, in writeback()
1811 op->data, in writeback()
1812 op->bytes * op->count); in writeback()
1814 kvm_write_sse_reg(op->addr.xmm, &op->vec_val); in writeback()
1817 kvm_write_mmx_reg(op->addr.mm, &op->mm_val); in writeback()
1832 rsp_increment(ctxt, -len); in emulate_push()
1842 ctxt->dst.type = OP_NONE; in em_push()
1843 return emulate_push(ctxt, &ctxt->src.val, ctxt->op_bytes); in em_push()
1864 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_pop()
1873 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulate_popf()
1874 int cpl = ctxt->ops->cpl(ctxt); in emulate_popf()
1885 switch(ctxt->mode) { in emulate_popf()
1899 default: /* real mode */ in emulate_popf()
1905 (ctxt->eflags & ~change_mask) | (val & change_mask); in emulate_popf()
1912 ctxt->dst.type = OP_REG; in em_popf()
1913 ctxt->dst.addr.reg = &ctxt->eflags; in em_popf()
1914 ctxt->dst.bytes = ctxt->op_bytes; in em_popf()
1915 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_popf()
1921 unsigned frame_size = ctxt->src.val; in em_enter()
1922 unsigned nesting_level = ctxt->src2.val & 31; in em_enter()
1935 reg_read(ctxt, VCPU_REGS_RSP) - frame_size, in em_enter()
1944 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes); in em_leave()
1949 int seg = ctxt->src2.val; in em_push_sreg()
1951 ctxt->src.val = get_segment_selector(ctxt, seg); in em_push_sreg()
1952 if (ctxt->op_bytes == 4) { in em_push_sreg()
1953 rsp_increment(ctxt, -2); in em_push_sreg()
1954 ctxt->op_bytes = 2; in em_push_sreg()
1962 int seg = ctxt->src2.val; in em_pop_sreg()
1971 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_pop_sreg()
1972 if (ctxt->op_bytes > 2) in em_pop_sreg()
1973 rsp_increment(ctxt, ctxt->op_bytes - 2); in em_pop_sreg()
1987 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg)); in em_pusha()
2001 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM; in em_pushf()
2013 rsp_increment(ctxt, ctxt->op_bytes); in em_popa()
2014 --reg; in em_popa()
2017 rc = emulate_pop(ctxt, &val, ctxt->op_bytes); in em_popa()
2020 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); in em_popa()
2021 --reg; in em_popa()
2028 const struct x86_emulate_ops *ops = ctxt->ops; in __emulate_int_real()
2036 ctxt->src.val = ctxt->eflags; in __emulate_int_real()
2041 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC); in __emulate_int_real()
2043 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); in __emulate_int_real()
2048 ctxt->src.val = ctxt->_eip; in __emulate_int_real()
2053 ops->get_idt(ctxt, &dt); in __emulate_int_real()
2070 ctxt->_eip = eip; in __emulate_int_real()
2088 switch(ctxt->mode) { in emulate_int()
2096 /* Protected mode interrupts unimplemented yet */ in emulate_int()
2118 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes); in emulate_iret_real()
2126 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in emulate_iret_real()
2131 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes); in emulate_iret_real()
2141 ctxt->_eip = temp_eip; in emulate_iret_real()
2143 if (ctxt->op_bytes == 4) in emulate_iret_real()
2144 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); in emulate_iret_real()
2145 else if (ctxt->op_bytes == 2) { in emulate_iret_real()
2146 ctxt->eflags &= ~0xffff; in emulate_iret_real()
2147 ctxt->eflags |= temp_eflags; in emulate_iret_real()
2150 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ in emulate_iret_real()
2151 ctxt->eflags |= X86_EFLAGS_FIXED; in emulate_iret_real()
2152 ctxt->ops->set_nmi_mask(ctxt, false); in emulate_iret_real()
2159 switch(ctxt->mode) { in em_iret()
2167 /* iret from protected mode unimplemented yet */ in em_iret()
2177 u8 cpl = ctxt->ops->cpl(ctxt); in em_jmp_far()
2179 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_jmp_far()
2187 rc = assign_eip_far(ctxt, ctxt->src.val); in em_jmp_far()
2197 return assign_eip_near(ctxt, ctxt->src.val); in em_jmp_abs()
2205 old_eip = ctxt->_eip; in em_call_near_abs()
2206 rc = assign_eip_near(ctxt, ctxt->src.val); in em_call_near_abs()
2209 ctxt->src.val = old_eip; in em_call_near_abs()
2216 u64 old = ctxt->dst.orig_val64; in em_cmpxchg8b()
2218 if (ctxt->dst.bytes == 16) in em_cmpxchg8b()
2225 ctxt->eflags &= ~X86_EFLAGS_ZF; in em_cmpxchg8b()
2227 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | in em_cmpxchg8b()
2230 ctxt->eflags |= X86_EFLAGS_ZF; in em_cmpxchg8b()
2240 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret()
2252 int cpl = ctxt->ops->cpl(ctxt); in em_ret_far()
2255 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_far()
2258 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in em_ret_far()
2281 rsp_increment(ctxt, ctxt->src.val); in em_ret_far_imm()
2288 ctxt->dst.orig_val = ctxt->dst.val; in em_cmpxchg()
2289 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2290 ctxt->src.orig_val = ctxt->src.val; in em_cmpxchg()
2291 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2294 if (ctxt->eflags & X86_EFLAGS_ZF) { in em_cmpxchg()
2296 ctxt->src.type = OP_NONE; in em_cmpxchg()
2297 ctxt->dst.val = ctxt->src.orig_val; in em_cmpxchg()
2300 ctxt->src.type = OP_REG; in em_cmpxchg()
2301 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2302 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2303 /* Create write-cycle to dest by writing the same value */ in em_cmpxchg()
2304 ctxt->dst.val = ctxt->dst.orig_val; in em_cmpxchg()
2311 int seg = ctxt->src2.val; in em_lseg()
2315 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_lseg()
2321 ctxt->dst.val = ctxt->src.val; in em_lseg()
2327 if (!ctxt->ops->is_smm(ctxt)) in em_rsm()
2330 if (ctxt->ops->leave_smm(ctxt)) in em_rsm()
2331 ctxt->ops->triple_fault(ctxt); in em_rsm()
2339 cs->l = 0; /* will be adjusted later */ in setup_syscalls_segments()
2341 cs->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2343 cs->type = 0x0b; /* Read, Execute, Accessed */ in setup_syscalls_segments()
2344 cs->s = 1; in setup_syscalls_segments()
2345 cs->dpl = 0; /* will be adjusted later */ in setup_syscalls_segments()
2346 cs->p = 1; in setup_syscalls_segments()
2347 cs->d = 1; in setup_syscalls_segments()
2348 cs->avl = 0; in setup_syscalls_segments()
2352 ss->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2353 ss->s = 1; in setup_syscalls_segments()
2354 ss->type = 0x03; /* Read/Write, Accessed */ in setup_syscalls_segments()
2355 ss->d = 1; /* 32bit stack segment */ in setup_syscalls_segments()
2356 ss->dpl = 0; in setup_syscalls_segments()
2357 ss->p = 1; in setup_syscalls_segments()
2358 ss->l = 0; in setup_syscalls_segments()
2359 ss->avl = 0; in setup_syscalls_segments()
2364 const struct x86_emulate_ops *ops = ctxt->ops; in em_syscall()
2370 /* syscall is not available in real mode */ in em_syscall()
2371 if (ctxt->mode == X86EMUL_MODE_REAL || in em_syscall()
2372 ctxt->mode == X86EMUL_MODE_VM86) in em_syscall()
2376 * Intel compatible CPUs only support SYSCALL in 64-bit mode, whereas in em_syscall()
2377 * AMD allows SYSCALL in any flavor of protected mode. Note, it's in em_syscall()
2379 * as SYSCALL won't fault in the "wrong" mode, i.e. there is no #UD in em_syscall()
2380 * for KVM to trap-and-emulate, unlike emulating AMD on Intel. in em_syscall()
2382 if (ctxt->mode != X86EMUL_MODE_PROT64 && in em_syscall()
2383 ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_syscall()
2386 ops->get_msr(ctxt, MSR_EFER, &efer); in em_syscall()
2391 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2400 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_syscall()
2401 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_syscall()
2403 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip; in em_syscall()
2406 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags; in em_syscall()
2408 ops->get_msr(ctxt, in em_syscall()
2409 ctxt->mode == X86EMUL_MODE_PROT64 ? in em_syscall()
2411 ctxt->_eip = msr_data; in em_syscall()
2413 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); in em_syscall()
2414 ctxt->eflags &= ~msr_data; in em_syscall()
2415 ctxt->eflags |= X86_EFLAGS_FIXED; in em_syscall()
2418 /* legacy mode */ in em_syscall()
2419 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2420 ctxt->_eip = (u32)msr_data; in em_syscall()
2422 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_syscall()
2425 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; in em_syscall()
2431 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysenter()
2437 ops->get_msr(ctxt, MSR_EFER, &efer); in em_sysenter()
2438 /* inject #GP if in real mode */ in em_sysenter()
2439 if (ctxt->mode == X86EMUL_MODE_REAL) in em_sysenter()
2443 * Intel's architecture allows SYSENTER in compatibility mode, but AMD in em_sysenter()
2444 * does not. Note, AMD does allow SYSENTER in legacy protected mode. in em_sysenter()
2446 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA) && in em_sysenter()
2447 !ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_sysenter()
2450 /* sysenter/sysexit have not been tested in 64bit mode. */ in em_sysenter()
2451 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_sysenter()
2454 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysenter()
2459 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_sysenter()
2467 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysenter()
2468 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysenter()
2470 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data); in em_sysenter()
2471 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data; in em_sysenter()
2473 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); in em_sysenter()
2477 ctxt->mode = X86EMUL_MODE_PROT64; in em_sysenter()
2484 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysexit()
2490 /* inject #GP if in real mode or Virtual 8086 mode */ in em_sysexit()
2491 if (ctxt->mode == X86EMUL_MODE_REAL || in em_sysexit()
2492 ctxt->mode == X86EMUL_MODE_VM86) in em_sysexit()
2497 if ((ctxt->rex_prefix & 0x8) != 0x0) in em_sysexit()
2507 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysexit()
2532 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysexit()
2533 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysexit()
2535 ctxt->_eip = rdx; in em_sysexit()
2536 ctxt->mode = usermode; in em_sysexit()
2545 if (ctxt->mode == X86EMUL_MODE_REAL) in emulator_bad_iopl()
2547 if (ctxt->mode == X86EMUL_MODE_VM86) in emulator_bad_iopl()
2549 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulator_bad_iopl()
2550 return ctxt->ops->cpl(ctxt) > iopl; in emulator_bad_iopl()
2559 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_io_port_access_allowed()
2564 unsigned mask = (1 << len) - 1; in emulator_io_port_access_allowed()
2575 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR); in emulator_io_port_access_allowed()
2584 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true); in emulator_io_port_access_allowed()
2589 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true); in emulator_io_port_access_allowed()
2600 if (ctxt->perm_ok) in emulator_io_permitted()
2607 ctxt->perm_ok = true; in emulator_io_permitted()
2616 * manner when ECX is zero due to REP-string optimizations. in string_registers_quirk()
2621 if (ctxt->ad_bytes != 4) in string_registers_quirk()
2625 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); in string_registers_quirk()
2631 switch (ctxt->b) { in string_registers_quirk()
2634 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1; in string_registers_quirk()
2638 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1; in string_registers_quirk()
2646 tss->ip = ctxt->_eip; in save_state_to_tss16()
2647 tss->flag = ctxt->eflags; in save_state_to_tss16()
2648 tss->ax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss16()
2649 tss->cx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss16()
2650 tss->dx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss16()
2651 tss->bx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss16()
2652 tss->sp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss16()
2653 tss->bp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss16()
2654 tss->si = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss16()
2655 tss->di = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss16()
2657 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss16()
2658 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss16()
2659 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss16()
2660 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss16()
2661 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR); in save_state_to_tss16()
2670 ctxt->_eip = tss->ip; in load_state_from_tss16()
2671 ctxt->eflags = tss->flag | 2; in load_state_from_tss16()
2672 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax; in load_state_from_tss16()
2673 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx; in load_state_from_tss16()
2674 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx; in load_state_from_tss16()
2675 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx; in load_state_from_tss16()
2676 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp; in load_state_from_tss16()
2677 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp; in load_state_from_tss16()
2678 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si; in load_state_from_tss16()
2679 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di; in load_state_from_tss16()
2685 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR); in load_state_from_tss16()
2686 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss16()
2687 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss16()
2688 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss16()
2689 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss16()
2691 cpl = tss->cs & 3; in load_state_from_tss16()
2697 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, in load_state_from_tss16()
2701 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss16()
2705 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss16()
2709 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss16()
2713 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss16()
2759 tss->eip = ctxt->_eip; in save_state_to_tss32()
2760 tss->eflags = ctxt->eflags; in save_state_to_tss32()
2761 tss->eax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss32()
2762 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss32()
2763 tss->edx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss32()
2764 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss32()
2765 tss->esp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss32()
2766 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss32()
2767 tss->esi = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss32()
2768 tss->edi = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss32()
2770 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss32()
2771 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss32()
2772 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss32()
2773 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss32()
2774 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS); in save_state_to_tss32()
2775 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS); in save_state_to_tss32()
2784 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3)) in load_state_from_tss32()
2786 ctxt->_eip = tss->eip; in load_state_from_tss32()
2787 ctxt->eflags = tss->eflags | 2; in load_state_from_tss32()
2790 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax; in load_state_from_tss32()
2791 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx; in load_state_from_tss32()
2792 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx; in load_state_from_tss32()
2793 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx; in load_state_from_tss32()
2794 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp; in load_state_from_tss32()
2795 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp; in load_state_from_tss32()
2796 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi; in load_state_from_tss32()
2797 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi; in load_state_from_tss32()
2804 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR); in load_state_from_tss32()
2805 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss32()
2806 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss32()
2807 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss32()
2808 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss32()
2809 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS); in load_state_from_tss32()
2810 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS); in load_state_from_tss32()
2813 * If we're switching between Protected Mode and VM86, we need to make in load_state_from_tss32()
2814 * sure to update the mode before loading the segment descriptors so in load_state_from_tss32()
2817 if (ctxt->eflags & X86_EFLAGS_VM) { in load_state_from_tss32()
2818 ctxt->mode = X86EMUL_MODE_VM86; in load_state_from_tss32()
2821 ctxt->mode = X86EMUL_MODE_PROT32; in load_state_from_tss32()
2822 cpl = tss->cs & 3; in load_state_from_tss32()
2829 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, in load_state_from_tss32()
2833 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss32()
2837 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss32()
2841 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss32()
2845 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss32()
2849 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, in load_state_from_tss32()
2853 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, in load_state_from_tss32()
2876 ldt_sel_offset - eip_offset); in task_switch_32()
2901 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_do_task_switch()
2906 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR); in emulator_do_task_switch()
2926 * 3. jmp/call to TSS/task-gate: No check is performed since the in emulator_do_task_switch()
2930 if (idt_index != -1) { in emulator_do_task_switch()
2941 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl) in emulator_do_task_switch()
2959 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; in emulator_do_task_switch()
2975 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT; in emulator_do_task_switch()
2982 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS); in emulator_do_task_switch()
2983 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR); in emulator_do_task_switch()
2986 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2; in emulator_do_task_switch()
2987 ctxt->lock_prefix = 0; in emulator_do_task_switch()
2988 ctxt->src.val = (unsigned long) error_code; in emulator_do_task_switch()
2992 dr7 = ops->get_dr(ctxt, 7); in emulator_do_task_switch()
2993 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN)); in emulator_do_task_switch()
3005 ctxt->_eip = ctxt->eip; in emulator_task_switch()
3006 ctxt->dst.type = OP_NONE; in emulator_task_switch()
3012 ctxt->eip = ctxt->_eip; in emulator_task_switch()
3020 struct operand *op) in string_addr_inc() argument
3022 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count; in string_addr_inc()
3024 register_address_increment(ctxt, reg, df * op->bytes); in string_addr_inc()
3025 op->addr.mem.ea = register_address(ctxt, reg); in string_addr_inc()
3033 cf = ctxt->eflags & X86_EFLAGS_CF; in em_das()
3034 al = ctxt->dst.val; in em_das()
3039 af = ctxt->eflags & X86_EFLAGS_AF; in em_das()
3041 al -= 6; in em_das()
3048 al -= 0x60; in em_das()
3052 ctxt->dst.val = al; in em_das()
3054 ctxt->src.type = OP_IMM; in em_das()
3055 ctxt->src.val = 0; in em_das()
3056 ctxt->src.bytes = 1; in em_das()
3058 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF); in em_das()
3060 ctxt->eflags |= X86_EFLAGS_CF; in em_das()
3062 ctxt->eflags |= X86_EFLAGS_AF; in em_das()
3070 if (ctxt->src.val == 0) in em_aam()
3073 al = ctxt->dst.val & 0xff; in em_aam()
3074 ah = al / ctxt->src.val; in em_aam()
3075 al %= ctxt->src.val; in em_aam()
3077 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8); in em_aam()
3080 ctxt->src.type = OP_IMM; in em_aam()
3081 ctxt->src.val = 0; in em_aam()
3082 ctxt->src.bytes = 1; in em_aam()
3090 u8 al = ctxt->dst.val & 0xff; in em_aad()
3091 u8 ah = (ctxt->dst.val >> 8) & 0xff; in em_aad()
3093 al = (al + (ah * ctxt->src.val)) & 0xff; in em_aad()
3095 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al; in em_aad()
3098 ctxt->src.type = OP_IMM; in em_aad()
3099 ctxt->src.val = 0; in em_aad()
3100 ctxt->src.bytes = 1; in em_aad()
3109 long rel = ctxt->src.val; in em_call()
3111 ctxt->src.val = (unsigned long)ctxt->_eip; in em_call()
3124 const struct x86_emulate_ops *ops = ctxt->ops; in em_call_far()
3125 int cpl = ctxt->ops->cpl(ctxt); in em_call_far()
3126 enum x86emul_mode prev_mode = ctxt->mode; in em_call_far()
3128 old_eip = ctxt->_eip; in em_call_far()
3129 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS); in em_call_far()
3131 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_call_far()
3137 rc = assign_eip_far(ctxt, ctxt->src.val); in em_call_far()
3141 ctxt->src.val = old_cs; in em_call_far()
3146 ctxt->src.val = old_eip; in em_call_far()
3156 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS); in em_call_far()
3157 ctxt->mode = prev_mode; in em_call_far()
3167 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_near_imm()
3173 rsp_increment(ctxt, ctxt->src.val); in em_ret_near_imm()
3180 ctxt->src.val = ctxt->dst.val; in em_xchg()
3181 write_register_operand(&ctxt->src); in em_xchg()
3184 ctxt->dst.val = ctxt->src.orig_val; in em_xchg()
3185 ctxt->lock_prefix = 1; in em_xchg()
3191 ctxt->dst.val = ctxt->src2.val; in em_imul_3op()
3197 ctxt->dst.type = OP_REG; in em_cwd()
3198 ctxt->dst.bytes = ctxt->src.bytes; in em_cwd()
3199 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in em_cwd()
3200 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); in em_cwd()
3209 if (!ctxt->ops->guest_has_rdpid(ctxt)) in em_rdpid()
3212 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux); in em_rdpid()
3213 ctxt->dst.val = tsc_aux; in em_rdpid()
3221 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); in em_rdtsc()
3231 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc)) in em_rdpmc()
3240 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr)); in em_mov()
3248 if (!ctxt->ops->guest_has_movbe(ctxt)) in em_movbe()
3251 switch (ctxt->op_bytes) { in em_movbe()
3258 * Both casting ->valptr and ->val to u16 breaks strict aliasing in em_movbe()
3261 tmp = (u16)ctxt->src.val; in em_movbe()
3262 ctxt->dst.val &= ~0xffffUL; in em_movbe()
3263 ctxt->dst.val |= (unsigned long)swab16(tmp); in em_movbe()
3266 ctxt->dst.val = swab32((u32)ctxt->src.val); in em_movbe()
3269 ctxt->dst.val = swab64(ctxt->src.val); in em_movbe()
3279 int cr_num = ctxt->modrm_reg; in em_cr_write()
3282 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val)) in em_cr_write()
3286 ctxt->dst.type = OP_NONE; in em_cr_write()
3291 * which can affect the cpu's execution mode. in em_cr_write()
3305 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_dr_write()
3306 val = ctxt->src.val & ~0ULL; in em_dr_write()
3308 val = ctxt->src.val & ~0U; in em_dr_write()
3311 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0) in em_dr_write()
3315 ctxt->dst.type = OP_NONE; in em_dr_write()
3327 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data); in em_wrmsr()
3341 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data); in em_rdmsr()
3356 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_store_sreg()
3357 ctxt->ops->cpl(ctxt) > 0) in em_store_sreg()
3360 ctxt->dst.val = get_segment_selector(ctxt, segment); in em_store_sreg()
3361 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM) in em_store_sreg()
3362 ctxt->dst.bytes = 2; in em_store_sreg()
3368 if (ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_rm_sreg()
3371 return em_store_sreg(ctxt, ctxt->modrm_reg); in em_mov_rm_sreg()
3376 u16 sel = ctxt->src.val; in em_mov_sreg_rm()
3378 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_sreg_rm()
3381 if (ctxt->modrm_reg == VCPU_SREG_SS) in em_mov_sreg_rm()
3382 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_mov_sreg_rm()
3385 ctxt->dst.type = OP_NONE; in em_mov_sreg_rm()
3386 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg); in em_mov_sreg_rm()
3396 u16 sel = ctxt->src.val; in em_lldt()
3399 ctxt->dst.type = OP_NONE; in em_lldt()
3410 u16 sel = ctxt->src.val; in em_ltr()
3413 ctxt->dst.type = OP_NONE; in em_ltr()
3423 rc = __linearize(ctxt, ctxt->src.addr.mem, &max_size, 1, ctxt->mode, in em_invlpg()
3426 ctxt->ops->invlpg(ctxt, linear); in em_invlpg()
3428 ctxt->dst.type = OP_NONE; in em_invlpg()
3436 cr0 = ctxt->ops->get_cr(ctxt, 0); in em_clts()
3438 ctxt->ops->set_cr(ctxt, 0, cr0); in em_clts()
3444 int rc = ctxt->ops->fix_hypercall(ctxt); in em_hypercall()
3449 /* Let the processor re-execute the fixed hypercall */ in em_hypercall()
3450 ctxt->_eip = ctxt->eip; in em_hypercall()
3452 ctxt->dst.type = OP_NONE; in em_hypercall()
3462 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in emulate_store_desc_ptr()
3463 ctxt->ops->cpl(ctxt) > 0) in emulate_store_desc_ptr()
3466 if (ctxt->mode == X86EMUL_MODE_PROT64) in emulate_store_desc_ptr()
3467 ctxt->op_bytes = 8; in emulate_store_desc_ptr()
3469 if (ctxt->op_bytes == 2) { in emulate_store_desc_ptr()
3470 ctxt->op_bytes = 4; in emulate_store_desc_ptr()
3474 ctxt->dst.type = OP_NONE; in emulate_store_desc_ptr()
3475 return segmented_write_std(ctxt, ctxt->dst.addr.mem, in emulate_store_desc_ptr()
3476 &desc_ptr, 2 + ctxt->op_bytes); in emulate_store_desc_ptr()
3481 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt); in em_sgdt()
3486 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt); in em_sidt()
3494 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_lgdt_lidt()
3495 ctxt->op_bytes = 8; in em_lgdt_lidt()
3496 rc = read_descriptor(ctxt, ctxt->src.addr.mem, in em_lgdt_lidt()
3498 ctxt->op_bytes); in em_lgdt_lidt()
3501 if (ctxt->mode == X86EMUL_MODE_PROT64 && in em_lgdt_lidt()
3506 ctxt->ops->set_gdt(ctxt, &desc_ptr); in em_lgdt_lidt()
3508 ctxt->ops->set_idt(ctxt, &desc_ptr); in em_lgdt_lidt()
3510 ctxt->dst.type = OP_NONE; in em_lgdt_lidt()
3526 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_smsw()
3527 ctxt->ops->cpl(ctxt) > 0) in em_smsw()
3530 if (ctxt->dst.type == OP_MEM) in em_smsw()
3531 ctxt->dst.bytes = 2; in em_smsw()
3532 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0); in em_smsw()
3538 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul) in em_lmsw()
3539 | (ctxt->src.val & 0x0f)); in em_lmsw()
3540 ctxt->dst.type = OP_NONE; in em_lmsw()
3548 register_address_increment(ctxt, VCPU_REGS_RCX, -1); in em_loop()
3550 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) in em_loop()
3551 rc = jmp_rel(ctxt, ctxt->src.val); in em_loop()
3561 rc = jmp_rel(ctxt, ctxt->src.val); in em_jcxz()
3568 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val, in em_in()
3569 &ctxt->dst.val)) in em_in()
3577 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val, in em_out()
3578 &ctxt->src.val, 1); in em_out()
3580 ctxt->dst.type = OP_NONE; in em_out()
3589 ctxt->eflags &= ~X86_EFLAGS_IF; in em_cli()
3598 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI; in em_sti()
3599 ctxt->eflags |= X86_EFLAGS_IF; in em_sti()
3608 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr); in em_cpuid()
3610 ctxt->ops->cpl(ctxt)) { in em_cpuid()
3616 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); in em_cpuid()
3632 ctxt->eflags &= ~0xffUL; in em_sahf()
3633 ctxt->eflags |= flags | X86_EFLAGS_FIXED; in em_sahf()
3640 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8; in em_lahf()
3646 switch (ctxt->op_bytes) { in em_bswap()
3649 asm("bswap %0" : "+r"(ctxt->dst.val)); in em_bswap()
3653 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val)); in em_bswap()
3673 ctxt->dst.val = (s32) ctxt->src.val; in em_movsxd()
3679 if (!ctxt->ops->guest_has_fxsr(ctxt)) in check_fxsr()
3682 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in check_fxsr()
3689 if (ctxt->mode >= X86EMUL_MODE_PROT64) in check_fxsr()
3696 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3707 if (ctxt->mode == X86EMUL_MODE_PROT64) in fxstate_size()
3710 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR; in fxstate_size()
3715 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3716 * 1) 16 bit mode
3717 * 2) 32 bit mode
3718 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3721 * 3) 64-bit mode with REX.W prefix
3722 * - like (2), but XMM 8-15 are being saved and restored
3723 * 4) 64-bit mode without REX.W prefix
3724 * - like (3), but FIP and FDP are 64 bit
3726 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3750 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, in em_fxsave()
3769 __fxstate_size(16) - used_size); in fxregs_fixup()
3785 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size); in em_fxrstor()
3815 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE)) in em_xsetbv()
3822 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax)) in em_xsetbv()
3842 if (!valid_cr(ctxt->modrm_reg)) in check_cr_access()
3850 int dr = ctxt->modrm_reg; in check_dr_read()
3856 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_dr_read()
3860 if (ctxt->ops->get_dr(ctxt, 7) & DR7_GD) { in check_dr_read()
3863 dr6 = ctxt->ops->get_dr(ctxt, 6); in check_dr_read()
3866 ctxt->ops->set_dr(ctxt, 6, dr6); in check_dr_read()
3875 u64 new_val = ctxt->src.val64; in check_dr_write()
3876 int dr = ctxt->modrm_reg; in check_dr_write()
3888 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in check_svme()
3909 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdtsc()
3911 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt)) in check_rdtsc()
3919 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdpmc()
3923 * VMware allows access to these Pseduo-PMCs even when read via RDPMC in check_rdpmc()
3932 * protected mode. in check_rdpmc()
3934 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || in check_rdpmc()
3935 ctxt->ops->check_rdpmc_early(ctxt, rcx)) in check_rdpmc()
3943 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u); in check_perm_in()
3944 if (!emulator_io_permitted(ctxt, ctxt->src.val, ctxt->dst.bytes)) in check_perm_in()
3952 ctxt->src.bytes = min(ctxt->src.bytes, 4u); in check_perm_out()
3953 if (!emulator_io_permitted(ctxt, ctxt->dst.val, ctxt->src.bytes)) in check_perm_out()
4172 /* 0xC0 - 0xC7 */
4174 /* 0xC8 - 0xCF */
4176 /* 0xD0 - 0xC7 */
4178 /* 0xD8 - 0xDF */
4180 /* 0xE0 - 0xE7 */
4182 /* 0xE8 - 0xEF */
4184 /* 0xF0 - 0xF7 */
4186 /* 0xF8 - 0xFF */
4193 /* 0xC0 - 0xC7 */
4195 /* 0xC8 - 0xCF */
4197 /* 0xD0 - 0xC7 */
4199 /* 0xD8 - 0xDF */
4201 /* 0xE0 - 0xE7 */
4203 /* 0xE8 - 0xEF */
4205 /* 0xF0 - 0xF7 */
4207 /* 0xF8 - 0xFF */
4214 /* 0xC0 - 0xC7 */
4216 /* 0xC8 - 0xCF */
4218 /* 0xD0 - 0xC7 */
4220 /* 0xD8 - 0xDF */
4222 /* 0xE0 - 0xE7 */
4224 /* 0xE8 - 0xEF */
4226 /* 0xF0 - 0xF7 */
4228 /* 0xF8 - 0xFF */
4245 /* 0x00 - 0x07 */
4249 /* 0x08 - 0x0F */
4253 /* 0x10 - 0x17 */
4257 /* 0x18 - 0x1F */
4261 /* 0x20 - 0x27 */
4263 /* 0x28 - 0x2F */
4265 /* 0x30 - 0x37 */
4267 /* 0x38 - 0x3F */
4269 /* 0x40 - 0x4F */
4271 /* 0x50 - 0x57 */
4273 /* 0x58 - 0x5F */
4275 /* 0x60 - 0x67 */
4280 /* 0x68 - 0x6F */
4287 /* 0x70 - 0x7F */
4289 /* 0x80 - 0x87 */
4296 /* 0x88 - 0x8F */
4303 /* 0x90 - 0x97 */
4305 /* 0x98 - 0x9F */
4311 /* 0xA0 - 0xA7 */
4316 /* 0xA8 - 0xAF */
4321 /* 0xB0 - 0xB7 */
4323 /* 0xB8 - 0xBF */
4325 /* 0xC0 - 0xC7 */
4332 /* 0xC8 - 0xCF */
4340 /* 0xD0 - 0xD7 */
4347 /* 0xD8 - 0xDF */
4349 /* 0xE0 - 0xE7 */
4354 /* 0xE8 - 0xEF */
4361 /* 0xF0 - 0xF7 */
4365 /* 0xF8 - 0xFF */
4372 /* 0x00 - 0x0F */
4378 /* 0x10 - 0x1F */
4388 /* 0x20 - 0x2F */
4400 /* 0x30 - 0x3F */
4409 /* 0x40 - 0x4F */
4411 /* 0x50 - 0x5F */
4413 /* 0x60 - 0x6F */
4418 /* 0x70 - 0x7F */
4423 /* 0x80 - 0x8F */
4425 /* 0x90 - 0x9F */
4427 /* 0xA0 - 0xA7 */
4433 /* 0xA8 - 0xAF */
4440 /* 0xB0 - 0xB7 */
4447 /* 0xB8 - 0xBF */
4454 /* 0xC0 - 0xC7 */
4458 /* 0xC8 - 0xCF */
4460 /* 0xD0 - 0xDF */
4462 /* 0xE0 - 0xEF */
4465 /* 0xF0 - 0xFF */
4490 /* 0x00 - 0x7f */
4492 /* 0x80 - 0xef */
4494 /* 0xf0 - 0xf1 */
4497 /* 0xf2 - 0xff */
4521 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in imm_size()
4527 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op, in decode_imm() argument
4532 op->type = OP_IMM; in decode_imm()
4533 op->bytes = size; in decode_imm()
4534 op->addr.mem.ea = ctxt->_eip; in decode_imm()
4535 /* NB. Immediates are sign-extended as necessary. */ in decode_imm()
4536 switch (op->bytes) { in decode_imm()
4538 op->val = insn_fetch(s8, ctxt); in decode_imm()
4541 op->val = insn_fetch(s16, ctxt); in decode_imm()
4544 op->val = insn_fetch(s32, ctxt); in decode_imm()
4547 op->val = insn_fetch(s64, ctxt); in decode_imm()
4551 switch (op->bytes) { in decode_imm()
4553 op->val &= 0xff; in decode_imm()
4556 op->val &= 0xffff; in decode_imm()
4559 op->val &= 0xffffffff; in decode_imm()
4567 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op, in decode_operand() argument
4574 decode_register_operand(ctxt, op); in decode_operand()
4577 rc = decode_imm(ctxt, op, 1, false); in decode_operand()
4580 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4582 *op = ctxt->memop; in decode_operand()
4583 ctxt->memopp = op; in decode_operand()
4584 if (ctxt->d & BitOp) in decode_operand()
4586 op->orig_val = op->val; in decode_operand()
4589 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8; in decode_operand()
4592 op->type = OP_REG; in decode_operand()
4593 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4594 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4595 fetch_register_operand(op); in decode_operand()
4596 op->orig_val = op->val; in decode_operand()
4599 op->type = OP_REG; in decode_operand()
4600 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes; in decode_operand()
4601 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4602 fetch_register_operand(op); in decode_operand()
4603 op->orig_val = op->val; in decode_operand()
4606 if (ctxt->d & ByteOp) { in decode_operand()
4607 op->type = OP_NONE; in decode_operand()
4610 op->type = OP_REG; in decode_operand()
4611 op->bytes = ctxt->op_bytes; in decode_operand()
4612 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4613 fetch_register_operand(op); in decode_operand()
4614 op->orig_val = op->val; in decode_operand()
4617 op->type = OP_MEM; in decode_operand()
4618 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4619 op->addr.mem.ea = in decode_operand()
4621 op->addr.mem.seg = VCPU_SREG_ES; in decode_operand()
4622 op->val = 0; in decode_operand()
4623 op->count = 1; in decode_operand()
4626 op->type = OP_REG; in decode_operand()
4627 op->bytes = 2; in decode_operand()
4628 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4629 fetch_register_operand(op); in decode_operand()
4632 op->type = OP_IMM; in decode_operand()
4633 op->bytes = 1; in decode_operand()
4634 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff; in decode_operand()
4637 rc = decode_imm(ctxt, op, 1, true); in decode_operand()
4640 op->type = OP_IMM; in decode_operand()
4641 op->bytes = 1; in decode_operand()
4642 op->val = 1; in decode_operand()
4645 rc = decode_imm(ctxt, op, imm_size(ctxt), true); in decode_operand()
4648 rc = decode_imm(ctxt, op, ctxt->op_bytes, true); in decode_operand()
4651 ctxt->memop.bytes = 1; in decode_operand()
4652 if (ctxt->memop.type == OP_REG) { in decode_operand()
4653 ctxt->memop.addr.reg = decode_register(ctxt, in decode_operand()
4654 ctxt->modrm_rm, true); in decode_operand()
4655 fetch_register_operand(&ctxt->memop); in decode_operand()
4659 ctxt->memop.bytes = 2; in decode_operand()
4662 ctxt->memop.bytes = 4; in decode_operand()
4665 rc = decode_imm(ctxt, op, 2, false); in decode_operand()
4668 rc = decode_imm(ctxt, op, imm_size(ctxt), false); in decode_operand()
4671 op->type = OP_MEM; in decode_operand()
4672 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4673 op->addr.mem.ea = in decode_operand()
4675 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4676 op->val = 0; in decode_operand()
4677 op->count = 1; in decode_operand()
4680 op->type = OP_MEM; in decode_operand()
4681 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4682 op->addr.mem.ea = in decode_operand()
4686 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4687 op->val = 0; in decode_operand()
4690 op->type = OP_IMM; in decode_operand()
4691 op->addr.mem.ea = ctxt->_eip; in decode_operand()
4692 op->bytes = ctxt->op_bytes + 2; in decode_operand()
4693 insn_fetch_arr(op->valptr, op->bytes, ctxt); in decode_operand()
4696 ctxt->memop.bytes = ctxt->op_bytes + 2; in decode_operand()
4699 op->type = OP_IMM; in decode_operand()
4700 op->val = VCPU_SREG_ES; in decode_operand()
4703 op->type = OP_IMM; in decode_operand()
4704 op->val = VCPU_SREG_CS; in decode_operand()
4707 op->type = OP_IMM; in decode_operand()
4708 op->val = VCPU_SREG_SS; in decode_operand()
4711 op->type = OP_IMM; in decode_operand()
4712 op->val = VCPU_SREG_DS; in decode_operand()
4715 op->type = OP_IMM; in decode_operand()
4716 op->val = VCPU_SREG_FS; in decode_operand()
4719 op->type = OP_IMM; in decode_operand()
4720 op->val = VCPU_SREG_GS; in decode_operand()
4725 op->type = OP_NONE; /* Disable writeback. */ in decode_operand()
4736 int mode = ctxt->mode; in x86_decode_insn() local
4744 ctxt->memop.type = OP_NONE; in x86_decode_insn()
4745 ctxt->memopp = NULL; in x86_decode_insn()
4746 ctxt->_eip = ctxt->eip; in x86_decode_insn()
4747 ctxt->fetch.ptr = ctxt->fetch.data; in x86_decode_insn()
4748 ctxt->fetch.end = ctxt->fetch.data + insn_len; in x86_decode_insn()
4749 ctxt->opcode_len = 1; in x86_decode_insn()
4750 ctxt->intercept = x86_intercept_none; in x86_decode_insn()
4752 memcpy(ctxt->fetch.data, insn, insn_len); in x86_decode_insn()
4759 switch (mode) { in x86_decode_insn()
4763 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS); in x86_decode_insn()
4783 ctxt->op_bytes = def_op_bytes; in x86_decode_insn()
4784 ctxt->ad_bytes = def_ad_bytes; in x86_decode_insn()
4788 switch (ctxt->b = insn_fetch(u8, ctxt)) { in x86_decode_insn()
4789 case 0x66: /* operand-size override */ in x86_decode_insn()
4792 ctxt->op_bytes = def_op_bytes ^ 6; in x86_decode_insn()
4794 case 0x67: /* address-size override */ in x86_decode_insn()
4795 if (mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4797 ctxt->ad_bytes = def_ad_bytes ^ 12; in x86_decode_insn()
4800 ctxt->ad_bytes = def_ad_bytes ^ 6; in x86_decode_insn()
4804 ctxt->seg_override = VCPU_SREG_ES; in x86_decode_insn()
4808 ctxt->seg_override = VCPU_SREG_CS; in x86_decode_insn()
4812 ctxt->seg_override = VCPU_SREG_SS; in x86_decode_insn()
4816 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
4820 ctxt->seg_override = VCPU_SREG_FS; in x86_decode_insn()
4824 ctxt->seg_override = VCPU_SREG_GS; in x86_decode_insn()
4827 if (mode != X86EMUL_MODE_PROT64) in x86_decode_insn()
4829 ctxt->rex_prefix = ctxt->b; in x86_decode_insn()
4832 ctxt->lock_prefix = 1; in x86_decode_insn()
4836 ctxt->rep_prefix = ctxt->b; in x86_decode_insn()
4844 ctxt->rex_prefix = 0; in x86_decode_insn()
4850 if (ctxt->rex_prefix & 8) in x86_decode_insn()
4851 ctxt->op_bytes = 8; /* REX.W */ in x86_decode_insn()
4854 opcode = opcode_table[ctxt->b]; in x86_decode_insn()
4855 /* Two-byte opcode? */ in x86_decode_insn()
4856 if (ctxt->b == 0x0f) { in x86_decode_insn()
4857 ctxt->opcode_len = 2; in x86_decode_insn()
4858 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4859 opcode = twobyte_table[ctxt->b]; in x86_decode_insn()
4862 if (ctxt->b == 0x38) { in x86_decode_insn()
4863 ctxt->opcode_len = 3; in x86_decode_insn()
4864 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4865 opcode = opcode_map_0f_38[ctxt->b]; in x86_decode_insn()
4868 ctxt->d = opcode.flags; in x86_decode_insn()
4870 if (ctxt->d & ModRM) in x86_decode_insn()
4871 ctxt->modrm = insn_fetch(u8, ctxt); in x86_decode_insn()
4873 /* vex-prefix instructions are not implemented */ in x86_decode_insn()
4874 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) && in x86_decode_insn()
4875 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) { in x86_decode_insn()
4876 ctxt->d = NotImpl; in x86_decode_insn()
4879 while (ctxt->d & GroupMask) { in x86_decode_insn()
4880 switch (ctxt->d & GroupMask) { in x86_decode_insn()
4882 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
4886 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
4887 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
4888 opcode = opcode.u.gdual->mod3[goffset]; in x86_decode_insn()
4890 opcode = opcode.u.gdual->mod012[goffset]; in x86_decode_insn()
4893 goffset = ctxt->modrm & 7; in x86_decode_insn()
4897 if (ctxt->rep_prefix && op_prefix) in x86_decode_insn()
4899 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix; in x86_decode_insn()
4901 case 0x00: opcode = opcode.u.gprefix->pfx_no; break; in x86_decode_insn()
4902 case 0x66: opcode = opcode.u.gprefix->pfx_66; break; in x86_decode_insn()
4903 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break; in x86_decode_insn()
4904 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break; in x86_decode_insn()
4908 if (ctxt->modrm > 0xbf) { in x86_decode_insn()
4909 size_t size = ARRAY_SIZE(opcode.u.esc->high); in x86_decode_insn()
4911 ctxt->modrm - 0xc0, size); in x86_decode_insn()
4913 opcode = opcode.u.esc->high[index]; in x86_decode_insn()
4915 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7]; in x86_decode_insn()
4919 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
4920 opcode = opcode.u.idual->mod3; in x86_decode_insn()
4922 opcode = opcode.u.idual->mod012; in x86_decode_insn()
4925 if (ctxt->mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4926 opcode = opcode.u.mdual->mode64; in x86_decode_insn()
4928 opcode = opcode.u.mdual->mode32; in x86_decode_insn()
4934 ctxt->d &= ~(u64)GroupMask; in x86_decode_insn()
4935 ctxt->d |= opcode.flags; in x86_decode_insn()
4938 ctxt->is_branch = opcode.flags & IsBranch; in x86_decode_insn()
4941 if (ctxt->d == 0) in x86_decode_insn()
4944 ctxt->execute = opcode.u.execute; in x86_decode_insn()
4947 likely(!(ctxt->d & EmulateOnUD))) in x86_decode_insn()
4950 if (unlikely(ctxt->d & in x86_decode_insn()
4957 ctxt->check_perm = opcode.check_perm; in x86_decode_insn()
4958 ctxt->intercept = opcode.intercept; in x86_decode_insn()
4960 if (ctxt->d & NotImpl) in x86_decode_insn()
4963 if (mode == X86EMUL_MODE_PROT64) { in x86_decode_insn()
4964 if (ctxt->op_bytes == 4 && (ctxt->d & Stack)) in x86_decode_insn()
4965 ctxt->op_bytes = 8; in x86_decode_insn()
4966 else if (ctxt->d & NearBranch) in x86_decode_insn()
4967 ctxt->op_bytes = 8; in x86_decode_insn()
4970 if (ctxt->d & Op3264) { in x86_decode_insn()
4971 if (mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
4972 ctxt->op_bytes = 8; in x86_decode_insn()
4974 ctxt->op_bytes = 4; in x86_decode_insn()
4977 if ((ctxt->d & No16) && ctxt->op_bytes == 2) in x86_decode_insn()
4978 ctxt->op_bytes = 4; in x86_decode_insn()
4980 if (ctxt->d & Sse) in x86_decode_insn()
4981 ctxt->op_bytes = 16; in x86_decode_insn()
4982 else if (ctxt->d & Mmx) in x86_decode_insn()
4983 ctxt->op_bytes = 8; in x86_decode_insn()
4987 if (ctxt->d & ModRM) { in x86_decode_insn()
4988 rc = decode_modrm(ctxt, &ctxt->memop); in x86_decode_insn()
4991 ctxt->seg_override = ctxt->modrm_seg; in x86_decode_insn()
4993 } else if (ctxt->d & MemAbs) in x86_decode_insn()
4994 rc = decode_abs(ctxt, &ctxt->memop); in x86_decode_insn()
4999 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
5001 ctxt->memop.addr.mem.seg = ctxt->seg_override; in x86_decode_insn()
5007 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask); in x86_decode_insn()
5015 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask); in x86_decode_insn()
5020 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask); in x86_decode_insn()
5022 if (ctxt->rip_relative && likely(ctxt->memopp)) in x86_decode_insn()
5023 ctxt->memopp->addr.mem.ea = address_mask(ctxt, in x86_decode_insn()
5024 ctxt->memopp->addr.mem.ea + ctxt->_eip); in x86_decode_insn()
5028 ctxt->have_exception = true; in x86_decode_insn()
5034 return ctxt->d & PageTable; in x86_page_table_writing_insn()
5043 * - if REPE/REPZ and ZF = 0 then done in string_insn_completed()
5044 * - if REPNE/REPNZ and ZF = 1 then done in string_insn_completed()
5046 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) || in string_insn_completed()
5047 (ctxt->b == 0xae) || (ctxt->b == 0xaf)) in string_insn_completed()
5048 && (((ctxt->rep_prefix == REPE_PREFIX) && in string_insn_completed()
5049 ((ctxt->eflags & X86_EFLAGS_ZF) == 0)) in string_insn_completed()
5050 || ((ctxt->rep_prefix == REPNE_PREFIX) && in string_insn_completed()
5051 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF)))) in string_insn_completed()
5071 static void fetch_possible_mmx_operand(struct operand *op) in fetch_possible_mmx_operand() argument
5073 if (op->type == OP_MM) in fetch_possible_mmx_operand()
5074 kvm_read_mmx_reg(op->addr.mm, &op->mm_val); in fetch_possible_mmx_operand()
5079 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; in fastop()
5081 if (!(ctxt->d & ByteOp)) in fastop()
5082 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE; in fastop()
5085 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags), in fastop()
5087 : "c"(ctxt->src2.val)); in fastop()
5089 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); in fastop()
5098 ctxt->rip_relative = false; in init_decode_cache()
5099 ctxt->rex_prefix = 0; in init_decode_cache()
5100 ctxt->lock_prefix = 0; in init_decode_cache()
5101 ctxt->rep_prefix = 0; in init_decode_cache()
5102 ctxt->regs_valid = 0; in init_decode_cache()
5103 ctxt->regs_dirty = 0; in init_decode_cache()
5105 ctxt->io_read.pos = 0; in init_decode_cache()
5106 ctxt->io_read.end = 0; in init_decode_cache()
5107 ctxt->mem_read.end = 0; in init_decode_cache()
5112 const struct x86_emulate_ops *ops = ctxt->ops; in x86_emulate_insn()
5114 int saved_dst_type = ctxt->dst.type; in x86_emulate_insn()
5115 bool is_guest_mode = ctxt->ops->is_guest_mode(ctxt); in x86_emulate_insn()
5117 ctxt->mem_read.pos = 0; in x86_emulate_insn()
5120 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) { in x86_emulate_insn()
5125 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) { in x86_emulate_insn()
5130 if (unlikely(ctxt->d & in x86_emulate_insn()
5132 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) || in x86_emulate_insn()
5133 (ctxt->d & Undefined)) { in x86_emulate_insn()
5138 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM))) in x86_emulate_insn()
5139 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) { in x86_emulate_insn()
5144 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) { in x86_emulate_insn()
5149 if (ctxt->d & Mmx) { in x86_emulate_insn()
5157 fetch_possible_mmx_operand(&ctxt->src); in x86_emulate_insn()
5158 fetch_possible_mmx_operand(&ctxt->src2); in x86_emulate_insn()
5159 if (!(ctxt->d & Mov)) in x86_emulate_insn()
5160 fetch_possible_mmx_operand(&ctxt->dst); in x86_emulate_insn()
5163 if (unlikely(is_guest_mode) && ctxt->intercept) { in x86_emulate_insn()
5164 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5170 /* Instruction can only be executed in protected mode */ in x86_emulate_insn()
5171 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) { in x86_emulate_insn()
5177 if ((ctxt->d & Priv) && ops->cpl(ctxt)) { in x86_emulate_insn()
5178 if (ctxt->d & PrivUD) in x86_emulate_insn()
5186 if (ctxt->d & CheckPerm) { in x86_emulate_insn()
5187 rc = ctxt->check_perm(ctxt); in x86_emulate_insn()
5192 if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5193 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5199 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5203 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5204 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5210 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) { in x86_emulate_insn()
5211 rc = segmented_read(ctxt, ctxt->src.addr.mem, in x86_emulate_insn()
5212 ctxt->src.valptr, ctxt->src.bytes); in x86_emulate_insn()
5215 ctxt->src.orig_val64 = ctxt->src.val64; in x86_emulate_insn()
5218 if (ctxt->src2.type == OP_MEM) { in x86_emulate_insn()
5219 rc = segmented_read(ctxt, ctxt->src2.addr.mem, in x86_emulate_insn()
5220 &ctxt->src2.val, ctxt->src2.bytes); in x86_emulate_insn()
5225 if ((ctxt->d & DstMask) == ImplicitOps) in x86_emulate_insn()
5229 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) { in x86_emulate_insn()
5230 /* optimisation - avoid slow emulated read if Mov */ in x86_emulate_insn()
5231 rc = segmented_read(ctxt, ctxt->dst.addr.mem, in x86_emulate_insn()
5232 &ctxt->dst.val, ctxt->dst.bytes); in x86_emulate_insn()
5234 if (!(ctxt->d & NoWrite) && in x86_emulate_insn()
5236 ctxt->exception.vector == PF_VECTOR) in x86_emulate_insn()
5237 ctxt->exception.error_code |= PFERR_WRITE_MASK; in x86_emulate_insn()
5241 /* Copy full 64-bit value for CMPXCHG8B. */ in x86_emulate_insn()
5242 ctxt->dst.orig_val64 = ctxt->dst.val64; in x86_emulate_insn()
5246 if (unlikely(is_guest_mode) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5247 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5253 if (ctxt->rep_prefix && (ctxt->d & String)) in x86_emulate_insn()
5254 ctxt->eflags |= X86_EFLAGS_RF; in x86_emulate_insn()
5256 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5258 if (ctxt->execute) { in x86_emulate_insn()
5259 if (ctxt->d & Fastop) in x86_emulate_insn()
5260 rc = fastop(ctxt, ctxt->fop); in x86_emulate_insn()
5262 rc = ctxt->execute(ctxt); in x86_emulate_insn()
5268 if (ctxt->opcode_len == 2) in x86_emulate_insn()
5270 else if (ctxt->opcode_len == 3) in x86_emulate_insn()
5273 switch (ctxt->b) { in x86_emulate_insn()
5275 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5276 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5279 ctxt->dst.val = ctxt->src.addr.mem.ea; in x86_emulate_insn()
5282 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX)) in x86_emulate_insn()
5283 ctxt->dst.type = OP_NONE; in x86_emulate_insn()
5288 switch (ctxt->op_bytes) { in x86_emulate_insn()
5289 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break; in x86_emulate_insn()
5290 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break; in x86_emulate_insn()
5291 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break; in x86_emulate_insn()
5298 rc = emulate_int(ctxt, ctxt->src.val); in x86_emulate_insn()
5301 if (ctxt->eflags & X86_EFLAGS_OF) in x86_emulate_insn()
5306 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5307 ctxt->dst.type = OP_NONE; /* Disable writeback. */ in x86_emulate_insn()
5310 ctxt->ops->halt(ctxt); in x86_emulate_insn()
5314 ctxt->eflags ^= X86_EFLAGS_CF; in x86_emulate_insn()
5317 ctxt->eflags &= ~X86_EFLAGS_CF; in x86_emulate_insn()
5320 ctxt->eflags |= X86_EFLAGS_CF; in x86_emulate_insn()
5323 ctxt->eflags &= ~X86_EFLAGS_DF; in x86_emulate_insn()
5326 ctxt->eflags |= X86_EFLAGS_DF; in x86_emulate_insn()
5336 if (ctxt->d & SrcWrite) { in x86_emulate_insn()
5337 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR); in x86_emulate_insn()
5338 rc = writeback(ctxt, &ctxt->src); in x86_emulate_insn()
5342 if (!(ctxt->d & NoWrite)) { in x86_emulate_insn()
5343 rc = writeback(ctxt, &ctxt->dst); in x86_emulate_insn()
5352 ctxt->dst.type = saved_dst_type; in x86_emulate_insn()
5354 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5355 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src); in x86_emulate_insn()
5357 if ((ctxt->d & DstMask) == DstDI) in x86_emulate_insn()
5358 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst); in x86_emulate_insn()
5360 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5362 struct read_cache *r = &ctxt->io_read; in x86_emulate_insn()
5363 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5364 count = ctxt->src.count; in x86_emulate_insn()
5366 count = ctxt->dst.count; in x86_emulate_insn()
5367 register_address_increment(ctxt, VCPU_REGS_RCX, -count); in x86_emulate_insn()
5371 * Re-enter guest when pio read ahead buffer is empty in x86_emulate_insn()
5374 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) && in x86_emulate_insn()
5375 (r->end == 0 || r->end != r->pos)) { in x86_emulate_insn()
5381 ctxt->mem_read.end = 0; in x86_emulate_insn()
5387 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5390 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5391 if (ctxt->mode != X86EMUL_MODE_PROT64) in x86_emulate_insn()
5392 ctxt->eip = (u32)ctxt->_eip; in x86_emulate_insn()
5396 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt)) in x86_emulate_insn()
5398 ctxt->have_exception = true; in x86_emulate_insn()
5409 switch (ctxt->b) { in x86_emulate_insn()
5411 (ctxt->ops->wbinvd)(ctxt); in x86_emulate_insn()
5419 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5422 ctxt->dst.val = ops->get_dr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5425 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5426 ctxt->dst.val = ctxt->src.val; in x86_emulate_insn()
5427 else if (ctxt->op_bytes != 4) in x86_emulate_insn()
5428 ctxt->dst.type = OP_NONE; /* no writeback */ in x86_emulate_insn()
5431 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5432 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5435 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags); in x86_emulate_insn()
5438 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5439 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val in x86_emulate_insn()
5440 : (u16) ctxt->src.val; in x86_emulate_insn()
5443 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5444 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val : in x86_emulate_insn()
5445 (s16) ctxt->src.val; in x86_emulate_insn()
5474 if (ctxt->rep_prefix && (ctxt->d & String)) in emulator_can_use_gpa()
5477 if (ctxt->d & TwoMemOp) in emulator_can_use_gpa()