#define MAGIC_NUM 0xdeadbeefdeadbeefUL
#define GS_BASE 0x400000

static void test_cr8(void)
{
	unsigned long src, dst;

	dst = 777;
	src = 3;
	asm volatile("mov %[src], %%cr8; mov %%cr8, %[dst]"
		     : [dst]"+r"(dst), [src]"+r"(src));
	report(dst == 3 && src == 3, "mov %%cr8");
}

static void test_push(void *mem)
{
	unsigned long tmp;
	unsigned long *stack_top = mem + 4096;
	unsigned long *new_stack_top;
	unsigned long memw = 0x123456789abcdeful;

	memset(mem, 0x55, (void *)stack_top - mem);

	asm volatile("mov %%rsp, %[tmp] \n\t"
		     "mov %[stack_top], %%rsp \n\t"
		     "pushq $-7 \n\t"
		     "pushq %[reg] \n\t"
		     "pushq (%[mem]) \n\t"
		     "pushq $-7070707 \n\t"
		     "mov %%rsp, %[new_stack_top] \n\t"
		     "mov %[tmp], %%rsp"
		     : [tmp]"=&r"(tmp), [new_stack_top]"=r"(new_stack_top)
		     : [stack_top]"r"(stack_top),
		       [reg]"r"(-17l), [mem]"r"(&memw)
		     : "memory");

	report(stack_top[-1] == -7ul, "push $imm8");
	report(stack_top[-2] == -17ul, "push %%reg");
	report(stack_top[-3] == 0x123456789abcdeful, "push mem");
	report(stack_top[-4] == -7070707, "push $imm");
}

static void test_pop(void *mem)
{
	unsigned long tmp, tmp3, rsp, rbp;
	unsigned long *stack_top = mem + 4096;
	unsigned long memw = 0x123456789abcdeful;
	static unsigned long tmp2;

	memset(mem, 0x55, (void *)stack_top - mem);

	asm volatile("pushq %[val] \n\t"
		     "popq (%[mem])"
		     : : [val]"m"(memw), [mem]"r"(mem) : "memory");
	report(*(unsigned long *)mem == memw, "pop mem");

	memw = 7 - memw;
	asm volatile("mov %%rsp, %[tmp] \n\t"
		     "mov %[stack_top], %%rsp \n\t"
		     "pushq %[val] \n\t"
		     "popq %[tmp2] \n\t"
		     "mov %[tmp], %%rsp"
		     : [tmp]"=&r"(tmp), [tmp2]"=m"(tmp2)
		     : [val]"r"(memw), [stack_top]"r"(stack_top)
		     : "memory");
	report(tmp2 == memw, "pop mem (2)");

	memw = 129443 - memw;
	asm volatile("mov %%rsp, %[tmp] \n\t"
		     "mov %[stack_top], %%rsp \n\t"
		     "pushq %[val] \n\t"
		     "popq %[tmp2] \n\t"
		     "mov %[tmp], %%rsp"
		     : [tmp]"=&r"(tmp), [tmp2]"=r"(tmp2)
		     : [val]"r"(memw), [stack_top]"r"(stack_top)
		     : "memory");
	report(tmp2 == memw, "pop reg");

	asm volatile("mov %%rsp, %[tmp] \n\t"
		     "mov %[stack_top], %%rsp \n\t"
		     "lea 1f(%%rip), %%rax \n\t"
		     "push %%rax \n\t"
		     "ret \n\t"
		     "2: jmp 2b \n\t"
		     "1: mov %[tmp], %%rsp"
		     : [tmp]"=&r"(tmp) : [stack_top]"r"(stack_top)
		     : "memory", "rax");
	report_pass("ret");

	stack_top[-1] = 0x778899;
	asm volatile("mov %[stack_top], %%r8 \n\t"
		     "mov %%rsp, %%r9 \n\t"
		     "xchg %%rbp, %%r8 \n\t"
		     "leave \n\t"
		     "xchg %%rsp, %%r9 \n\t"
		     "xchg %%rbp, %%r8 \n\t"
		     "mov %%r9, %[tmp] \n\t"
		     "mov %%r8, %[tmp3]"
		     : [tmp]"=&r"(tmp), [tmp3]"=&r"(tmp3) : [stack_top]"r"(stack_top-1)
		     : "memory", "r8", "r9");
	report(tmp == (ulong)stack_top && tmp3 == 0x778899, "leave");

	rbp = 0xaa55aa55bb66bb66ULL;
	rsp = (unsigned long)stack_top;
	asm volatile("mov %[rsp], %%r8 \n\t"
		     "mov %[rbp], %%r9 \n\t"
		     "xchg %%rsp, %%r8 \n\t"
		     "xchg %%rbp, %%r9 \n\t"
		     "enter $0x1238, $0 \n\t"
		     "xchg %%rsp, %%r8 \n\t"
		     "xchg %%rbp, %%r9 \n\t"
		     "xchg %%r8, %[rsp] \n\t"
		     "xchg %%r9, %[rbp]"
		     : [rsp]"+a"(rsp), [rbp]"+b"(rbp) : : "memory", "r8", "r9");
	report(rsp == (unsigned long)stack_top - 8 - 0x1238
	       && rbp == (unsigned long)stack_top - 8
	       && stack_top[-1] == 0xaa55aa55bb66bb66ULL,
	       "enter");
}

static void test_ljmp(void *mem)
{
	unsigned char *m = mem;
	volatile int res = 1;

	*(unsigned long**)m = &&jmpf;
	asm volatile ("data16 mov %%cs, %0":"=m"(*(m + sizeof(unsigned long))));
	asm volatile ("rex64 ljmp *%0"::"m"(*m));
	res = 0;
jmpf:
	report(res, "ljmp");
}

static void test_xchg(void *mem)
{
	unsigned long *memq = mem;
	unsigned long rax;

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xchg %%al, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0xfedcba98765432ef && *memq == 0x123456789abcd10,
	       "xchg reg, r/m (1)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xchg %%ax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0xfedcba987654cdef && *memq == 0x123456789ab3210,
	       "xchg reg, r/m (2)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xchg %%eax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0x89abcdef && *memq == 0x123456776543210,
	       "xchg reg, r/m (3)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xchg %%rax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0x123456789abcdef && *memq == 0xfedcba9876543210,
	       "xchg reg, r/m (4)");
}

static void test_xadd(void *mem)
{
	unsigned long *memq = mem;
	unsigned long rax;

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xadd %%al, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0xfedcba98765432ef && *memq == 0x123456789abcdff,
	       "xadd reg, r/m (1)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xadd %%ax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0xfedcba987654cdef && *memq == 0x123456789abffff,
	       "xadd reg, r/m (2)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xadd %%eax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0x89abcdef && *memq == 0x1234567ffffffff,
	       "xadd reg, r/m (3)");

	asm volatile("mov $0x123456789abcdef, %%rax\n\t"
		     "mov %%rax, (%[memq])\n\t"
		     "mov $0xfedcba9876543210, %%rax\n\t"
		     "xadd %%rax, (%[memq])\n\t"
		     "mov %%rax, %[rax]\n\t"
		     : [rax]"=r"(rax)
		     : [memq]"r"(memq)
		     : "memory", "rax");
	report(rax == 0x123456789abcdef && *memq == 0xffffffffffffffff,
	       "xadd reg, r/m (4)");
}

static void test_muldiv(long *mem)
{
	long a, d, aa, dd;
	u8 ex = 1;

	*mem = 0; a = 1; d = 2;
	asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:"
	     : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem));
	report(a == 1 && d == 2 && ex, "divq (fault)");

	*mem = 987654321098765UL; a = 123456789012345UL; d = 123456789012345UL;
	asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:"
	     : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem));
	report(a == 0x1ffffffb1b963b33ul && d == 0x273ba4384ede2ul && !ex, "divq (1)");

	aa = 0x1111111111111111; dd = 0x2222222222222222;
	*mem = 0x3333333333333333; a = aa; d = dd;
	asm("mulb %2" : "+a"(a), "+d"(d) : "m"(*mem));
	report(a == 0x1111111111110363 && d == dd, "mulb mem");
	*mem = 0x3333333333333333; a = aa; d = dd;
	asm("mulw %2" : "+a"(a), "+d"(d) : "m"(*mem));
	report(a == 0x111111111111c963 && d == 0x2222222222220369, "mulw mem");
	*mem = 0x3333333333333333; a = aa; d = dd;
	asm("mull %2" : "+a"(a), "+d"(d) : "m"(*mem));
	report(a == 0x962fc963 && d == 0x369d036, "mull mem");
	*mem = 0x3333333333333333; a = aa; d = dd;
	asm("mulq %2" : "+a"(a), "+d"(d) : "m"(*mem));
	report(a == 0x2fc962fc962fc963 && d == 0x369d0369d0369d0, "mulq mem");
}

static void test_mmx(uint64_t *mem)
{
	uint64_t v;

	write_cr0(read_cr0() & ~6); /* EM, TS */
	asm volatile("fninit");
	v = 0x0102030405060708ULL;
	asm("movq %1, %0" : "=m"(*mem) : "y"(v));
	report(v == *mem, "movq (mmx, read)");
	*mem = 0x8070605040302010ull;
	asm("movq %1, %0" : "=y"(v) : "m"(*mem));
	report(v == *mem, "movq (mmx, write)");
}

static void test_rip_relative(unsigned *mem, char *insn_ram)
{
	/* movb $1, mem+2(%rip) */
	insn_ram[0] = 0xc6;
	insn_ram[1] = 0x05;
	*(unsigned *)&insn_ram[2] = 2 + (char *)mem - (insn_ram + 7);
	insn_ram[6] = 0x01;
	/* ret */
	insn_ram[7] = 0xc3;

	*mem = 0;
	asm("callq *%1" : "+m"(*mem) : "r"(insn_ram));
	report(*mem == 0x10000, "movb $imm, 0(%%rip)");
}

static void test_cmov(u32 *mem)
{
	u64 val;
	*mem = 0xabcdef12u;
	asm ("movq $0x1234567812345678, %%rax\n\t"
	     "cmpl %%eax, %%eax\n\t"
	     "cmovnel (%[mem]), %%eax\n\t"
	     "movq %%rax, %[val]\n\t"
	     : [val]"=r"(val) : [mem]"r"(mem) : "%rax", "cc");
	report(val == 0x12345678ul, "cmovnel");
}


static void test_mmx_movq_mf(uint64_t *mem)
{
	uint16_t fcw = 0;  /* all exceptions unmasked */
	uint64_t val;

	write_cr0(read_cr0() & ~(X86_CR0_TS | X86_CR0_EM));
	asm volatile("fninit\n\t"
		     "fldcw %[fcw]\n\t"
		     "fldz\n\t"
		     "fldz\n\t"
		     /* generate exception (0.0 / 0.0) */
		     "fdivp\n\t"
		     /* trigger #MF */
		     ASM_TRY_FEP("1f") "movq %%mm0, %[val]\n\t"
		     /* exit MMX mode */
		     "1: fnclex\n\t"
		     "emms\n\t"
		     : [val]"=m"(val)
		     : [fcw]"m"(fcw));
	report(exception_vector() == MF_VECTOR, "movq mmx generates #MF");
}

static void test_jmp_noncanonical(uint64_t *mem)
{
	*mem = NONCANONICAL;
	asm volatile (ASM_TRY("1f") "jmp *%0; 1:" : : "m"(*mem));
	report(exception_vector() == GP_VECTOR,
	       "jump to non-canonical address");
}

static void test_reg_noncanonical(void)
{
	/* RAX based, should #GP(0) */
	asm volatile(ASM_TRY("1f") "orq $0, (%[noncanonical]); 1:"
		     : : [noncanonical]"a"(NONCANONICAL));
	report(exception_vector() == GP_VECTOR && exception_error_code() == 0,
	       "non-canonical memory access, should %s(0), got %s(%u)",
	       exception_mnemonic(GP_VECTOR),
	       exception_mnemonic(exception_vector()), exception_error_code());

	/* RSP based, should #SS(0) */
	asm volatile(ASM_TRY("1f") "orq $0, (%%rsp,%[noncanonical],1); 1:"
		     : : [noncanonical]"r"(NONCANONICAL));
	report(exception_vector() == SS_VECTOR && exception_error_code() == 0,
	       "non-canonical rsp-based access, should %s(0), got %s(%u)",
	       exception_mnemonic(SS_VECTOR),
	       exception_mnemonic(exception_vector()), exception_error_code());

	/* RBP based, should #SS(0) */
	asm volatile(ASM_TRY("1f") "orq $0, (%%rbp,%[noncanonical],1); 1:"
		     : : [noncanonical]"r"(NONCANONICAL));
	report(exception_vector() == SS_VECTOR && exception_error_code() == 0,
	       "non-canonical rbp-based access, should %s(0), got %s(%u)",
	       exception_mnemonic(SS_VECTOR),
	       exception_mnemonic(exception_vector()), exception_error_code());
}

static void test_movabs(uint64_t *mem)
{
	/* mov $0x9090909090909090, %rcx */
	unsigned long rcx;
	asm(KVM_FEP "mov $0x9090909090909090, %0" : "=c" (rcx) : "0" (0));
	report(rcx == 0x9090909090909090, "64-bit mov imm2");
}

static void load_dpl0_seg(void)
{
	asm volatile(KVM_FEP "mov %0, %%fs" :: "r" (KERNEL_CS)); /* RPL=0 */
}

static void test_user_load_dpl0_seg(void)
{
	bool raised_vector;

	run_in_user((usermode_func)load_dpl0_seg, GP_VECTOR, 0, 0, 0, 0,
		    &raised_vector);

	report(raised_vector, "Wanted #GP on CPL=3 DPL=0 segment load");
}

static void test_push16(uint64_t *mem)
{
	uint64_t rsp1, rsp2;
	uint16_t r;

	asm volatile (	"movq %%rsp, %[rsp1]\n\t"
			"pushw %[v]\n\t"
			"popw %[r]\n\t"
			"movq %%rsp, %[rsp2]\n\t"
			"movq %[rsp1], %%rsp\n\t" :
			[rsp1]"=r"(rsp1), [rsp2]"=r"(rsp2), [r]"=r"(r)
			: [v]"m"(*mem) : "memory");
	report(rsp1 == rsp2, "push16");
}

static void test_sreg(volatile uint16_t *mem)
{
	u16 ss = read_ss();

	// check for null segment load
	*mem = 0;
	asm volatile("mov %0, %%ss" : : "m"(*mem));
	report(read_ss() == 0, "mov null, %%ss");

	// check for exception when ss.rpl != cpl on null segment load
	*mem = 3;
	asm volatile(ASM_TRY("1f") "mov %0, %%ss; 1:" : : "m"(*mem));
	report(exception_vector() == GP_VECTOR &&
	       exception_error_code() == 0 && read_ss() == 0,
	       "mov null, %%ss (with ss.rpl != cpl)");

	// check for exception when ss.rpl != cpl on non-null segment load
	*mem = KERNEL_DS | 3;
	asm volatile(ASM_TRY("1f") "mov %0, %%ss; 1:" : : "m"(*mem));
	report(exception_vector() == GP_VECTOR &&
	       exception_error_code() == KERNEL_DS && read_ss() == 0,
	       "mov non-null, %%ss (with ss.rpl != cpl)");

	write_ss(ss);
}

static uint64_t usr_gs_mov(void)
{
	static uint64_t dummy = MAGIC_NUM;
	uint64_t dummy_ptr = (uint64_t)&dummy;
	uint64_t ret;

	dummy_ptr -= GS_BASE;
	asm volatile("mov %%gs:(%1), %0" : "=r"(ret) : "r"(dummy_ptr));

	return ret;
}

static void test_iret(void)
{
	uint64_t val;
	bool raised_vector;

	/* Update GS base to 4MiB */
	wrmsr(MSR_GS_BASE, GS_BASE);

	/*
	* Per the SDM, jumping to user mode via `iret`, which is returning to
	* outer privilege level, for segment registers (ES, FS, GS, and DS)
	* if the check fails, the segment selector becomes null.
	*
	* In our test case, GS becomes null.
	*/
	val = run_in_user((usermode_func)usr_gs_mov, GP_VECTOR,
			0, 0, 0, 0, &raised_vector);

	report(val == MAGIC_NUM, "Test ret/iret with a nullified segment");
}

static void test_emulator_64(void *mem)
{
	void *insn_page = alloc_page();
	void *insn_ram  = vmap(virt_to_phys(insn_page), 4096);

	test_push(mem);
	test_pop(mem);

	test_xchg(mem);
	test_xadd(mem);

	test_cr8();

	test_ljmp(mem);
	test_muldiv(mem);
	test_mmx(mem);
	test_rip_relative(mem, insn_ram);
	test_iret();
	test_sreg(mem);
	test_cmov(mem);

	if (is_fep_available()) {
		test_mmx_movq_mf(mem);
		test_movabs(mem);
		test_user_load_dpl0_seg();
	}

	test_push16(mem);

	test_reg_noncanonical();
	test_jmp_noncanonical(mem);
}