1 #include "ioram.h" 2 #include "vm.h" 3 #include "libcflat.h" 4 #include "desc.h" 5 #include "types.h" 6 #include "processor.h" 7 #include "vmalloc.h" 8 #include "alloc_page.h" 9 10 #define memset __builtin_memset 11 #define TESTDEV_IO_PORT 0xe0 12 13 static int exceptions; 14 15 /* Forced emulation prefix, used to invoke the emulator unconditionally. */ 16 #define KVM_FEP "ud2; .byte 'k', 'v', 'm';" 17 #define KVM_FEP_LENGTH 5 18 static int fep_available = 1; 19 20 struct regs { 21 u64 rax, rbx, rcx, rdx; 22 u64 rsi, rdi, rsp, rbp; 23 u64 r8, r9, r10, r11; 24 u64 r12, r13, r14, r15; 25 u64 rip, rflags; 26 }; 27 struct regs inregs, outregs, save; 28 29 struct insn_desc { 30 u64 ptr; 31 size_t len; 32 }; 33 34 static char st1[] = "abcdefghijklmnop"; 35 36 static void test_stringio(void) 37 { 38 unsigned char r = 0; 39 asm volatile("cld \n\t" 40 "movw %0, %%dx \n\t" 41 "rep outsb \n\t" 42 : : "i"((short)TESTDEV_IO_PORT), 43 "S"(st1), "c"(sizeof(st1) - 1)); 44 asm volatile("inb %1, %0\n\t" : "=a"(r) : "i"((short)TESTDEV_IO_PORT)); 45 report("outsb up", r == st1[sizeof(st1) - 2]); /* last char */ 46 47 asm volatile("std \n\t" 48 "movw %0, %%dx \n\t" 49 "rep outsb \n\t" 50 : : "i"((short)TESTDEV_IO_PORT), 51 "S"(st1 + sizeof(st1) - 2), "c"(sizeof(st1) - 1)); 52 asm volatile("cld \n\t" : : ); 53 asm volatile("in %1, %0\n\t" : "=a"(r) : "i"((short)TESTDEV_IO_PORT)); 54 report("outsb down", r == st1[0]); 55 } 56 57 static void test_cmps_one(unsigned char *m1, unsigned char *m3) 58 { 59 void *rsi, *rdi; 60 long rcx, tmp; 61 62 rsi = m1; rdi = m3; rcx = 30; 63 asm volatile("xor %[tmp], %[tmp] \n\t" 64 "repe/cmpsb" 65 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 66 : : "cc"); 67 report("repe/cmpsb (1)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 68 69 rsi = m1; rdi = m3; rcx = 30; 70 asm volatile("or $1, %[tmp]\n\t" // clear ZF 71 "repe/cmpsb" 72 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 73 : : "cc"); 74 report("repe/cmpsb (1.zf)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 75 76 rsi = m1; rdi = m3; rcx = 15; 77 asm volatile("xor %[tmp], %[tmp] \n\t" 78 "repe/cmpsw" 79 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 80 : : "cc"); 81 report("repe/cmpsw (1)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 82 83 rsi = m1; rdi = m3; rcx = 7; 84 asm volatile("xor %[tmp], %[tmp] \n\t" 85 "repe/cmpsl" 86 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 87 : : "cc"); 88 report("repe/cmpll (1)", rcx == 0 && rsi == m1 + 28 && rdi == m3 + 28); 89 90 rsi = m1; rdi = m3; rcx = 4; 91 asm volatile("xor %[tmp], %[tmp] \n\t" 92 "repe/cmpsq" 93 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 94 : : "cc"); 95 report("repe/cmpsq (1)", rcx == 0 && rsi == m1 + 32 && rdi == m3 + 32); 96 97 rsi = m1; rdi = m3; rcx = 130; 98 asm volatile("xor %[tmp], %[tmp] \n\t" 99 "repe/cmpsb" 100 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 101 : : "cc"); 102 report("repe/cmpsb (2)", 103 rcx == 29 && rsi == m1 + 101 && rdi == m3 + 101); 104 105 rsi = m1; rdi = m3; rcx = 65; 106 asm volatile("xor %[tmp], %[tmp] \n\t" 107 "repe/cmpsw" 108 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 109 : : "cc"); 110 report("repe/cmpsw (2)", 111 rcx == 14 && rsi == m1 + 102 && rdi == m3 + 102); 112 113 rsi = m1; rdi = m3; rcx = 32; 114 asm volatile("xor %[tmp], %[tmp] \n\t" 115 "repe/cmpsl" 116 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 117 : : "cc"); 118 report("repe/cmpll (2)", 119 rcx == 6 && rsi == m1 + 104 && rdi == m3 + 104); 120 121 rsi = m1; rdi = m3; rcx = 16; 122 asm volatile("xor %[tmp], %[tmp] \n\t" 123 "repe/cmpsq" 124 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 125 : : "cc"); 126 report("repe/cmpsq (2)", 127 rcx == 3 && rsi == m1 + 104 && rdi == m3 + 104); 128 129 } 130 131 static void test_cmps(void *mem) 132 { 133 unsigned char *m1 = mem, *m2 = mem + 1024; 134 unsigned char m3[1024]; 135 136 for (int i = 0; i < 100; ++i) 137 m1[i] = m2[i] = m3[i] = i; 138 for (int i = 100; i < 200; ++i) 139 m1[i] = (m3[i] = m2[i] = i) + 1; 140 test_cmps_one(m1, m3); 141 test_cmps_one(m1, m2); 142 } 143 144 static void test_scas(void *mem) 145 { 146 bool z; 147 void *di; 148 149 *(ulong *)mem = 0x77665544332211; 150 151 di = mem; 152 asm ("scasb; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff11)); 153 report("scasb match", di == mem + 1 && z); 154 155 di = mem; 156 asm ("scasb; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff54)); 157 report("scasb mismatch", di == mem + 1 && !z); 158 159 di = mem; 160 asm ("scasw; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff2211)); 161 report("scasw match", di == mem + 2 && z); 162 163 di = mem; 164 asm ("scasw; setz %0" : "=rm"(z), "+D"(di) : "a"(0xffdd11)); 165 report("scasw mismatch", di == mem + 2 && !z); 166 167 di = mem; 168 asm ("scasl; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff44332211ul)); 169 report("scasd match", di == mem + 4 && z); 170 171 di = mem; 172 asm ("scasl; setz %0" : "=rm"(z), "+D"(di) : "a"(0x45332211)); 173 report("scasd mismatch", di == mem + 4 && !z); 174 175 di = mem; 176 asm ("scasq; setz %0" : "=rm"(z), "+D"(di) : "a"(0x77665544332211ul)); 177 report("scasq match", di == mem + 8 && z); 178 179 di = mem; 180 asm ("scasq; setz %0" : "=rm"(z), "+D"(di) : "a"(3)); 181 report("scasq mismatch", di == mem + 8 && !z); 182 } 183 184 static void test_cr8(void) 185 { 186 unsigned long src, dst; 187 188 dst = 777; 189 src = 3; 190 asm volatile("mov %[src], %%cr8; mov %%cr8, %[dst]" 191 : [dst]"+r"(dst), [src]"+r"(src)); 192 report("mov %%cr8", dst == 3 && src == 3); 193 } 194 195 static void test_push(void *mem) 196 { 197 unsigned long tmp; 198 unsigned long *stack_top = mem + 4096; 199 unsigned long *new_stack_top; 200 unsigned long memw = 0x123456789abcdeful; 201 202 memset(mem, 0x55, (void *)stack_top - mem); 203 204 asm volatile("mov %%rsp, %[tmp] \n\t" 205 "mov %[stack_top], %%rsp \n\t" 206 "pushq $-7 \n\t" 207 "pushq %[reg] \n\t" 208 "pushq (%[mem]) \n\t" 209 "pushq $-7070707 \n\t" 210 "mov %%rsp, %[new_stack_top] \n\t" 211 "mov %[tmp], %%rsp" 212 : [tmp]"=&r"(tmp), [new_stack_top]"=r"(new_stack_top) 213 : [stack_top]"r"(stack_top), 214 [reg]"r"(-17l), [mem]"r"(&memw) 215 : "memory"); 216 217 report("push $imm8", stack_top[-1] == -7ul); 218 report("push %%reg", stack_top[-2] == -17ul); 219 report("push mem", stack_top[-3] == 0x123456789abcdeful); 220 report("push $imm", stack_top[-4] == -7070707); 221 } 222 223 static void test_pop(void *mem) 224 { 225 unsigned long tmp, tmp3, rsp, rbp; 226 unsigned long *stack_top = mem + 4096; 227 unsigned long memw = 0x123456789abcdeful; 228 static unsigned long tmp2; 229 230 memset(mem, 0x55, (void *)stack_top - mem); 231 232 asm volatile("pushq %[val] \n\t" 233 "popq (%[mem])" 234 : : [val]"m"(memw), [mem]"r"(mem) : "memory"); 235 report("pop mem", *(unsigned long *)mem == memw); 236 237 memw = 7 - memw; 238 asm volatile("mov %%rsp, %[tmp] \n\t" 239 "mov %[stack_top], %%rsp \n\t" 240 "pushq %[val] \n\t" 241 "popq %[tmp2] \n\t" 242 "mov %[tmp], %%rsp" 243 : [tmp]"=&r"(tmp), [tmp2]"=m"(tmp2) 244 : [val]"r"(memw), [stack_top]"r"(stack_top) 245 : "memory"); 246 report("pop mem (2)", tmp2 == memw); 247 248 memw = 129443 - memw; 249 asm volatile("mov %%rsp, %[tmp] \n\t" 250 "mov %[stack_top], %%rsp \n\t" 251 "pushq %[val] \n\t" 252 "popq %[tmp2] \n\t" 253 "mov %[tmp], %%rsp" 254 : [tmp]"=&r"(tmp), [tmp2]"=r"(tmp2) 255 : [val]"r"(memw), [stack_top]"r"(stack_top) 256 : "memory"); 257 report("pop reg", tmp2 == memw); 258 259 asm volatile("mov %%rsp, %[tmp] \n\t" 260 "mov %[stack_top], %%rsp \n\t" 261 "push $1f \n\t" 262 "ret \n\t" 263 "2: jmp 2b \n\t" 264 "1: mov %[tmp], %%rsp" 265 : [tmp]"=&r"(tmp) : [stack_top]"r"(stack_top) 266 : "memory"); 267 report("ret", 1); 268 269 stack_top[-1] = 0x778899; 270 asm volatile("mov %[stack_top], %%r8 \n\t" 271 "mov %%rsp, %%r9 \n\t" 272 "xchg %%rbp, %%r8 \n\t" 273 "leave \n\t" 274 "xchg %%rsp, %%r9 \n\t" 275 "xchg %%rbp, %%r8 \n\t" 276 "mov %%r9, %[tmp] \n\t" 277 "mov %%r8, %[tmp3]" 278 : [tmp]"=&r"(tmp), [tmp3]"=&r"(tmp3) : [stack_top]"r"(stack_top-1) 279 : "memory", "r8", "r9"); 280 report("leave", tmp == (ulong)stack_top && tmp3 == 0x778899); 281 282 rbp = 0xaa55aa55bb66bb66ULL; 283 rsp = (unsigned long)stack_top; 284 asm volatile("mov %[rsp], %%r8 \n\t" 285 "mov %[rbp], %%r9 \n\t" 286 "xchg %%rsp, %%r8 \n\t" 287 "xchg %%rbp, %%r9 \n\t" 288 "enter $0x1238, $0 \n\t" 289 "xchg %%rsp, %%r8 \n\t" 290 "xchg %%rbp, %%r9 \n\t" 291 "xchg %%r8, %[rsp] \n\t" 292 "xchg %%r9, %[rbp]" 293 : [rsp]"+a"(rsp), [rbp]"+b"(rbp) : : "memory", "r8", "r9"); 294 report("enter", 295 rsp == (unsigned long)stack_top - 8 - 0x1238 296 && rbp == (unsigned long)stack_top - 8 297 && stack_top[-1] == 0xaa55aa55bb66bb66ULL); 298 } 299 300 static void test_ljmp(void *mem) 301 { 302 unsigned char *m = mem; 303 volatile int res = 1; 304 305 *(unsigned long**)m = &&jmpf; 306 asm volatile ("data16/mov %%cs, %0":"=m"(*(m + sizeof(unsigned long)))); 307 asm volatile ("rex64/ljmp *%0"::"m"(*m)); 308 res = 0; 309 jmpf: 310 report("ljmp", res); 311 } 312 313 static void test_incdecnotneg(void *mem) 314 { 315 unsigned long *m = mem, v = 1234; 316 unsigned char *mb = mem, vb = 66; 317 318 *m = 0; 319 320 asm volatile ("incl %0":"+m"(*m)); 321 report("incl", *m == 1); 322 asm volatile ("decl %0":"+m"(*m)); 323 report("decl", *m == 0); 324 asm volatile ("incb %0":"+m"(*m)); 325 report("incb", *m == 1); 326 asm volatile ("decb %0":"+m"(*m)); 327 report("decb", *m == 0); 328 329 asm volatile ("lock incl %0":"+m"(*m)); 330 report("lock incl", *m == 1); 331 asm volatile ("lock decl %0":"+m"(*m)); 332 report("lock decl", *m == 0); 333 asm volatile ("lock incb %0":"+m"(*m)); 334 report("lock incb", *m == 1); 335 asm volatile ("lock decb %0":"+m"(*m)); 336 report("lock decb", *m == 0); 337 338 *m = v; 339 340 asm ("lock negq %0" : "+m"(*m)); v = -v; 341 report("lock negl", *m == v); 342 asm ("lock notq %0" : "+m"(*m)); v = ~v; 343 report("lock notl", *m == v); 344 345 *mb = vb; 346 347 asm ("lock negb %0" : "+m"(*mb)); vb = -vb; 348 report("lock negb", *mb == vb); 349 asm ("lock notb %0" : "+m"(*mb)); vb = ~vb; 350 report("lock notb", *mb == vb); 351 } 352 353 static void test_smsw(uint64_t *h_mem) 354 { 355 char mem[16]; 356 unsigned short msw, msw_orig, *pmsw; 357 int i, zero; 358 359 msw_orig = read_cr0(); 360 361 asm("smsw %0" : "=r"(msw)); 362 report("smsw (1)", msw == msw_orig); 363 364 memset(mem, 0, 16); 365 pmsw = (void *)mem; 366 asm("smsw %0" : "=m"(pmsw[4])); 367 zero = 1; 368 for (i = 0; i < 8; ++i) 369 if (i != 4 && pmsw[i]) 370 zero = 0; 371 report("smsw (2)", msw == pmsw[4] && zero); 372 373 /* Trigger exit on smsw */ 374 *h_mem = 0x12345678abcdeful; 375 asm volatile("smsw %0" : "+m"(*h_mem)); 376 report("smsw (3)", msw == (unsigned short)*h_mem && 377 (*h_mem & ~0xfffful) == 0x12345678ab0000ul); 378 } 379 380 static void test_lmsw(void) 381 { 382 char mem[16]; 383 unsigned short msw, *pmsw; 384 unsigned long cr0; 385 386 cr0 = read_cr0(); 387 388 msw = cr0 ^ 8; 389 asm("lmsw %0" : : "r"(msw)); 390 printf("before %lx after %lx\n", cr0, read_cr0()); 391 report("lmsw (1)", (cr0 ^ read_cr0()) == 8); 392 393 pmsw = (void *)mem; 394 *pmsw = cr0; 395 asm("lmsw %0" : : "m"(*pmsw)); 396 printf("before %lx after %lx\n", cr0, read_cr0()); 397 report("lmsw (2)", cr0 == read_cr0()); 398 399 /* lmsw can't clear cr0.pe */ 400 msw = (cr0 & ~1ul) ^ 4; /* change EM to force trap */ 401 asm("lmsw %0" : : "r"(msw)); 402 report("lmsw (3)", (cr0 ^ read_cr0()) == 4 && (cr0 & 1)); 403 404 /* back to normal */ 405 msw = cr0; 406 asm("lmsw %0" : : "r"(msw)); 407 } 408 409 static void test_xchg(void *mem) 410 { 411 unsigned long *memq = mem; 412 unsigned long rax; 413 414 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 415 "mov %%rax, (%[memq])\n\t" 416 "mov $0xfedcba9876543210, %%rax\n\t" 417 "xchg %%al, (%[memq])\n\t" 418 "mov %%rax, %[rax]\n\t" 419 : [rax]"=r"(rax) 420 : [memq]"r"(memq) 421 : "memory", "rax"); 422 report("xchg reg, r/m (1)", 423 rax == 0xfedcba98765432ef && *memq == 0x123456789abcd10); 424 425 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 426 "mov %%rax, (%[memq])\n\t" 427 "mov $0xfedcba9876543210, %%rax\n\t" 428 "xchg %%ax, (%[memq])\n\t" 429 "mov %%rax, %[rax]\n\t" 430 : [rax]"=r"(rax) 431 : [memq]"r"(memq) 432 : "memory", "rax"); 433 report("xchg reg, r/m (2)", 434 rax == 0xfedcba987654cdef && *memq == 0x123456789ab3210); 435 436 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 437 "mov %%rax, (%[memq])\n\t" 438 "mov $0xfedcba9876543210, %%rax\n\t" 439 "xchg %%eax, (%[memq])\n\t" 440 "mov %%rax, %[rax]\n\t" 441 : [rax]"=r"(rax) 442 : [memq]"r"(memq) 443 : "memory", "rax"); 444 report("xchg reg, r/m (3)", 445 rax == 0x89abcdef && *memq == 0x123456776543210); 446 447 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 448 "mov %%rax, (%[memq])\n\t" 449 "mov $0xfedcba9876543210, %%rax\n\t" 450 "xchg %%rax, (%[memq])\n\t" 451 "mov %%rax, %[rax]\n\t" 452 : [rax]"=r"(rax) 453 : [memq]"r"(memq) 454 : "memory", "rax"); 455 report("xchg reg, r/m (4)", 456 rax == 0x123456789abcdef && *memq == 0xfedcba9876543210); 457 } 458 459 static void test_xadd(void *mem) 460 { 461 unsigned long *memq = mem; 462 unsigned long rax; 463 464 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 465 "mov %%rax, (%[memq])\n\t" 466 "mov $0xfedcba9876543210, %%rax\n\t" 467 "xadd %%al, (%[memq])\n\t" 468 "mov %%rax, %[rax]\n\t" 469 : [rax]"=r"(rax) 470 : [memq]"r"(memq) 471 : "memory", "rax"); 472 report("xadd reg, r/m (1)", 473 rax == 0xfedcba98765432ef && *memq == 0x123456789abcdff); 474 475 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 476 "mov %%rax, (%[memq])\n\t" 477 "mov $0xfedcba9876543210, %%rax\n\t" 478 "xadd %%ax, (%[memq])\n\t" 479 "mov %%rax, %[rax]\n\t" 480 : [rax]"=r"(rax) 481 : [memq]"r"(memq) 482 : "memory", "rax"); 483 report("xadd reg, r/m (2)", 484 rax == 0xfedcba987654cdef && *memq == 0x123456789abffff); 485 486 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 487 "mov %%rax, (%[memq])\n\t" 488 "mov $0xfedcba9876543210, %%rax\n\t" 489 "xadd %%eax, (%[memq])\n\t" 490 "mov %%rax, %[rax]\n\t" 491 : [rax]"=r"(rax) 492 : [memq]"r"(memq) 493 : "memory", "rax"); 494 report("xadd reg, r/m (3)", 495 rax == 0x89abcdef && *memq == 0x1234567ffffffff); 496 497 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 498 "mov %%rax, (%[memq])\n\t" 499 "mov $0xfedcba9876543210, %%rax\n\t" 500 "xadd %%rax, (%[memq])\n\t" 501 "mov %%rax, %[rax]\n\t" 502 : [rax]"=r"(rax) 503 : [memq]"r"(memq) 504 : "memory", "rax"); 505 report("xadd reg, r/m (4)", 506 rax == 0x123456789abcdef && *memq == 0xffffffffffffffff); 507 } 508 509 static void test_btc(void *mem) 510 { 511 unsigned int *a = mem; 512 513 memset(mem, 0, 4 * sizeof(unsigned int)); 514 515 asm ("btcl $32, %0" :: "m"(a[0]) : "memory"); 516 asm ("btcl $1, %0" :: "m"(a[1]) : "memory"); 517 asm ("btcl %1, %0" :: "m"(a[0]), "r"(66) : "memory"); 518 report("btcl imm8, r/m", a[0] == 1 && a[1] == 2 && a[2] == 4); 519 520 asm ("btcl %1, %0" :: "m"(a[3]), "r"(-1) : "memory"); 521 report("btcl reg, r/m", a[0] == 1 && a[1] == 2 && a[2] == 0x80000004); 522 523 asm ("btcq %1, %0" : : "m"(a[2]), "r"(-1l) : "memory"); 524 report("btcq reg, r/m", a[0] == 1 && a[1] == 0x80000002 && 525 a[2] == 0x80000004 && a[3] == 0); 526 } 527 528 static void test_bsfbsr(void *mem) 529 { 530 unsigned long rax, *memq = mem; 531 unsigned eax, *meml = mem; 532 unsigned short ax, *memw = mem; 533 unsigned char z; 534 535 *memw = 0xc000; 536 asm("bsfw %[mem], %[a]" : [a]"=a"(ax) : [mem]"m"(*memw)); 537 report("bsfw r/m, reg", ax == 14); 538 539 *meml = 0xc0000000; 540 asm("bsfl %[mem], %[a]" : [a]"=a"(eax) : [mem]"m"(*meml)); 541 report("bsfl r/m, reg", eax == 30); 542 543 *memq = 0xc00000000000; 544 asm("bsfq %[mem], %[a]" : [a]"=a"(rax) : [mem]"m"(*memq)); 545 report("bsfq r/m, reg", rax == 46); 546 547 *memq = 0; 548 asm("bsfq %[mem], %[a]; setz %[z]" 549 : [a]"=a"(rax), [z]"=rm"(z) : [mem]"m"(*memq)); 550 report("bsfq r/m, reg", z == 1); 551 552 *memw = 0xc000; 553 asm("bsrw %[mem], %[a]" : [a]"=a"(ax) : [mem]"m"(*memw)); 554 report("bsrw r/m, reg", ax == 15); 555 556 *meml = 0xc0000000; 557 asm("bsrl %[mem], %[a]" : [a]"=a"(eax) : [mem]"m"(*meml)); 558 report("bsrl r/m, reg", eax == 31); 559 560 *memq = 0xc00000000000; 561 asm("bsrq %[mem], %[a]" : [a]"=a"(rax) : [mem]"m"(*memq)); 562 report("bsrq r/m, reg", rax == 47); 563 564 *memq = 0; 565 asm("bsrq %[mem], %[a]; setz %[z]" 566 : [a]"=a"(rax), [z]"=rm"(z) : [mem]"m"(*memq)); 567 report("bsrq r/m, reg", z == 1); 568 } 569 570 static void test_imul(ulong *mem) 571 { 572 ulong a; 573 574 *mem = 51; a = 0x1234567812345678UL; 575 asm ("imulw %1, %%ax" : "+a"(a) : "m"(*mem)); 576 report("imul ax, mem", a == 0x12345678123439e8); 577 578 *mem = 51; a = 0x1234567812345678UL; 579 asm ("imull %1, %%eax" : "+a"(a) : "m"(*mem)); 580 report("imul eax, mem", a == 0xa06d39e8); 581 582 *mem = 51; a = 0x1234567812345678UL; 583 asm ("imulq %1, %%rax" : "+a"(a) : "m"(*mem)); 584 report("imul rax, mem", a == 0xA06D39EBA06D39E8UL); 585 586 *mem = 0x1234567812345678UL; a = 0x8765432187654321L; 587 asm ("imulw $51, %1, %%ax" : "+a"(a) : "m"(*mem)); 588 report("imul ax, mem, imm8", a == 0x87654321876539e8); 589 590 *mem = 0x1234567812345678UL; 591 asm ("imull $51, %1, %%eax" : "+a"(a) : "m"(*mem)); 592 report("imul eax, mem, imm8", a == 0xa06d39e8); 593 594 *mem = 0x1234567812345678UL; 595 asm ("imulq $51, %1, %%rax" : "+a"(a) : "m"(*mem)); 596 report("imul rax, mem, imm8", a == 0xA06D39EBA06D39E8UL); 597 598 *mem = 0x1234567812345678UL; a = 0x8765432187654321L; 599 asm ("imulw $311, %1, %%ax" : "+a"(a) : "m"(*mem)); 600 report("imul ax, mem, imm", a == 0x8765432187650bc8); 601 602 *mem = 0x1234567812345678UL; 603 asm ("imull $311, %1, %%eax" : "+a"(a) : "m"(*mem)); 604 report("imul eax, mem, imm", a == 0x1d950bc8); 605 606 *mem = 0x1234567812345678UL; 607 asm ("imulq $311, %1, %%rax" : "+a"(a) : "m"(*mem)); 608 report("imul rax, mem, imm", a == 0x1D950BDE1D950BC8L); 609 } 610 611 static void test_muldiv(long *mem) 612 { 613 long a, d, aa, dd; 614 u8 ex = 1; 615 616 *mem = 0; a = 1; d = 2; 617 asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:" 618 : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem)); 619 report("divq (fault)", a == 1 && d == 2 && ex); 620 621 *mem = 987654321098765UL; a = 123456789012345UL; d = 123456789012345UL; 622 asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:" 623 : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem)); 624 report("divq (1)", 625 a == 0x1ffffffb1b963b33ul && d == 0x273ba4384ede2ul && !ex); 626 aa = 0x1111111111111111; dd = 0x2222222222222222; 627 *mem = 0x3333333333333333; a = aa; d = dd; 628 asm("mulb %2" : "+a"(a), "+d"(d) : "m"(*mem)); 629 report("mulb mem", a == 0x1111111111110363 && d == dd); 630 *mem = 0x3333333333333333; a = aa; d = dd; 631 asm("mulw %2" : "+a"(a), "+d"(d) : "m"(*mem)); 632 report("mulw mem", a == 0x111111111111c963 && d == 0x2222222222220369); 633 *mem = 0x3333333333333333; a = aa; d = dd; 634 asm("mull %2" : "+a"(a), "+d"(d) : "m"(*mem)); 635 report("mull mem", a == 0x962fc963 && d == 0x369d036); 636 *mem = 0x3333333333333333; a = aa; d = dd; 637 asm("mulq %2" : "+a"(a), "+d"(d) : "m"(*mem)); 638 report("mulq mem", a == 0x2fc962fc962fc963 && d == 0x369d0369d0369d0); 639 } 640 641 typedef unsigned __attribute__((vector_size(16))) sse128; 642 643 typedef union { 644 sse128 sse; 645 unsigned u[4]; 646 } sse_union; 647 648 static bool sseeq(sse_union *v1, sse_union *v2) 649 { 650 bool ok = true; 651 int i; 652 653 for (i = 0; i < 4; ++i) { 654 ok &= v1->u[i] == v2->u[i]; 655 } 656 657 return ok; 658 } 659 660 static __attribute__((target("sse"))) void test_sse(sse_union *mem) 661 { 662 sse_union v; 663 664 write_cr0(read_cr0() & ~6); /* EM, TS */ 665 write_cr4(read_cr4() | 0x200); /* OSFXSR */ 666 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 667 asm("movdqu %1, %0" : "=m"(*mem) : "x"(v.sse)); 668 report("movdqu (read)", sseeq(&v, mem)); 669 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 670 asm("movdqu %1, %0" : "=x"(v.sse) : "m"(*mem)); 671 report("movdqu (write)", sseeq(mem, &v)); 672 673 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 674 asm("movaps %1, %0" : "=m"(*mem) : "x"(v.sse)); 675 report("movaps (read)", sseeq(mem, &v)); 676 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 677 asm("movaps %1, %0" : "=x"(v.sse) : "m"(*mem)); 678 report("movaps (write)", sseeq(&v, mem)); 679 680 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 681 asm("movapd %1, %0" : "=m"(*mem) : "x"(v.sse)); 682 report("movapd (read)", sseeq(mem, &v)); 683 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 684 asm("movapd %1, %0" : "=x"(v.sse) : "m"(*mem)); 685 report("movapd (write)", sseeq(&v, mem)); 686 } 687 688 static void test_mmx(uint64_t *mem) 689 { 690 uint64_t v; 691 692 write_cr0(read_cr0() & ~6); /* EM, TS */ 693 asm volatile("fninit"); 694 v = 0x0102030405060708ULL; 695 asm("movq %1, %0" : "=m"(*mem) : "y"(v)); 696 report("movq (mmx, read)", v == *mem); 697 *mem = 0x8070605040302010ull; 698 asm("movq %1, %0" : "=y"(v) : "m"(*mem)); 699 report("movq (mmx, write)", v == *mem); 700 } 701 702 static void test_rip_relative(unsigned *mem, char *insn_ram) 703 { 704 /* movb $1, mem+2(%rip) */ 705 insn_ram[0] = 0xc6; 706 insn_ram[1] = 0x05; 707 *(unsigned *)&insn_ram[2] = 2 + (char *)mem - (insn_ram + 7); 708 insn_ram[6] = 0x01; 709 /* ret */ 710 insn_ram[7] = 0xc3; 711 712 *mem = 0; 713 asm("callq *%1" : "+m"(*mem) : "r"(insn_ram)); 714 report("movb $imm, 0(%%rip)", *mem == 0x10000); 715 } 716 717 static void test_shld_shrd(u32 *mem) 718 { 719 *mem = 0x12345678; 720 asm("shld %2, %1, %0" : "+m"(*mem) : "r"(0xaaaaaaaaU), "c"((u8)3)); 721 report("shld (cl)", *mem == ((0x12345678 << 3) | 5)); 722 *mem = 0x12345678; 723 asm("shrd %2, %1, %0" : "+m"(*mem) : "r"(0x55555555U), "c"((u8)3)); 724 report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29))); 725 } 726 727 static void test_cmov(u32 *mem) 728 { 729 u64 val; 730 *mem = 0xabcdef12u; 731 asm ("movq $0x1234567812345678, %%rax\n\t" 732 "cmpl %%eax, %%eax\n\t" 733 "cmovnel (%[mem]), %%eax\n\t" 734 "movq %%rax, %[val]\n\t" 735 : [val]"=r"(val) : [mem]"r"(mem) : "%rax", "cc"); 736 report("cmovnel", val == 0x12345678ul); 737 } 738 739 static unsigned long rip_advance; 740 741 static void advance_rip_and_note_exception(struct ex_regs *regs) 742 { 743 ++exceptions; 744 regs->rip += rip_advance; 745 } 746 747 static void test_mmx_movq_mf(uint64_t *mem) 748 { 749 /* movq %mm0, (%rax) */ 750 extern char movq_start, movq_end; 751 752 uint16_t fcw = 0; /* all exceptions unmasked */ 753 write_cr0(read_cr0() & ~6); /* TS, EM */ 754 exceptions = 0; 755 handle_exception(MF_VECTOR, advance_rip_and_note_exception); 756 asm volatile("fninit; fldcw %0" : : "m"(fcw)); 757 asm volatile("fldz; fldz; fdivp"); /* generate exception */ 758 759 rip_advance = &movq_end - &movq_start; 760 asm(KVM_FEP "movq_start: movq %mm0, (%rax); movq_end:"); 761 /* exit MMX mode */ 762 asm volatile("fnclex; emms"); 763 report("movq mmx generates #MF", exceptions == 1); 764 handle_exception(MF_VECTOR, 0); 765 } 766 767 static void test_jmp_noncanonical(uint64_t *mem) 768 { 769 extern char nc_jmp_start, nc_jmp_end; 770 771 *mem = 0x1111111111111111ul; 772 773 exceptions = 0; 774 rip_advance = &nc_jmp_end - &nc_jmp_start; 775 handle_exception(GP_VECTOR, advance_rip_and_note_exception); 776 asm volatile ("nc_jmp_start: jmp *%0; nc_jmp_end:" : : "m"(*mem)); 777 report("jump to non-canonical address", exceptions == 1); 778 handle_exception(GP_VECTOR, 0); 779 } 780 781 static void test_movabs(uint64_t *mem) 782 { 783 /* mov $0x9090909090909090, %rcx */ 784 unsigned long rcx; 785 asm(KVM_FEP "mov $0x9090909090909090, %0" : "=c" (rcx) : "0" (0)); 786 report("64-bit mov imm2", rcx == 0x9090909090909090); 787 } 788 789 static void test_smsw_reg(uint64_t *mem) 790 { 791 unsigned long cr0 = read_cr0(); 792 unsigned long rax; 793 const unsigned long in_rax = 0x1234567890abcdeful; 794 795 asm(KVM_FEP "smsww %w0\n\t" : "=a" (rax) : "0" (in_rax)); 796 report("16-bit smsw reg", (u16)rax == (u16)cr0 && 797 rax >> 16 == in_rax >> 16); 798 799 asm(KVM_FEP "smswl %k0\n\t" : "=a" (rax) : "0" (in_rax)); 800 report("32-bit smsw reg", rax == (u32)cr0); 801 802 asm(KVM_FEP "smswq %d0\n\t" : "=a" (rax) : "0" (in_rax)); 803 report("64-bit smsw reg", rax == cr0); 804 } 805 806 static void test_nop(uint64_t *mem) 807 { 808 unsigned long rax; 809 const unsigned long in_rax = 0x1234567890abcdeful; 810 asm(KVM_FEP "nop\n\t" : "=a" (rax) : "0" (in_rax)); 811 report("nop", rax == in_rax); 812 } 813 814 static void test_mov_dr(uint64_t *mem) 815 { 816 unsigned long rax; 817 const unsigned long in_rax = 0; 818 bool rtm_support = this_cpu_has(X86_FEATURE_RTM); 819 unsigned long dr6_fixed_1 = rtm_support ? 0xfffe0ff0ul : 0xffff0ff0ul; 820 asm(KVM_FEP "movq %0, %%dr6\n\t" 821 KVM_FEP "movq %%dr6, %0\n\t" : "=a" (rax) : "a" (in_rax)); 822 report("mov_dr6", rax == dr6_fixed_1); 823 } 824 825 static void test_push16(uint64_t *mem) 826 { 827 uint64_t rsp1, rsp2; 828 uint16_t r; 829 830 asm volatile ( "movq %%rsp, %[rsp1]\n\t" 831 "pushw %[v]\n\t" 832 "popw %[r]\n\t" 833 "movq %%rsp, %[rsp2]\n\t" 834 "movq %[rsp1], %%rsp\n\t" : 835 [rsp1]"=r"(rsp1), [rsp2]"=r"(rsp2), [r]"=r"(r) 836 : [v]"m"(*mem) : "memory"); 837 report("push16", rsp1 == rsp2); 838 } 839 840 static void test_crosspage_mmio(volatile uint8_t *mem) 841 { 842 volatile uint16_t w, *pw; 843 844 pw = (volatile uint16_t *)&mem[4095]; 845 mem[4095] = 0x99; 846 mem[4096] = 0x77; 847 asm volatile("mov %1, %0" : "=r"(w) : "m"(*pw) : "memory"); 848 report("cross-page mmio read", w == 0x7799); 849 asm volatile("mov %1, %0" : "=m"(*pw) : "r"((uint16_t)0x88aa)); 850 report("cross-page mmio write", mem[4095] == 0xaa && mem[4096] == 0x88); 851 } 852 853 static void test_string_io_mmio(volatile uint8_t *mem) 854 { 855 /* Cross MMIO pages.*/ 856 volatile uint8_t *mmio = mem + 4032; 857 858 asm volatile("outw %%ax, %%dx \n\t" : : "a"(0x9999), "d"(TESTDEV_IO_PORT)); 859 860 asm volatile ("cld; rep insb" : : "d" (TESTDEV_IO_PORT), "D" (mmio), "c" (1024)); 861 862 report("string_io_mmio", mmio[1023] == 0x99); 863 } 864 865 /* kvm doesn't allow lidt/lgdt from mmio, so the test is disabled */ 866 #if 0 867 static void test_lgdt_lidt(volatile uint8_t *mem) 868 { 869 struct descriptor_table_ptr orig, fresh = {}; 870 871 sgdt(&orig); 872 *(struct descriptor_table_ptr *)mem = (struct descriptor_table_ptr) { 873 .limit = 0xf234, 874 .base = 0x12345678abcd, 875 }; 876 cli(); 877 asm volatile("lgdt %0" : : "m"(*(struct descriptor_table_ptr *)mem)); 878 sgdt(&fresh); 879 lgdt(&orig); 880 sti(); 881 report("lgdt (long address)", orig.limit == fresh.limit && orig.base == fresh.base); 882 883 sidt(&orig); 884 *(struct descriptor_table_ptr *)mem = (struct descriptor_table_ptr) { 885 .limit = 0x432f, 886 .base = 0xdbca87654321, 887 }; 888 cli(); 889 asm volatile("lidt %0" : : "m"(*(struct descriptor_table_ptr *)mem)); 890 sidt(&fresh); 891 lidt(&orig); 892 sti(); 893 report("lidt (long address)", orig.limit == fresh.limit && orig.base == fresh.base); 894 } 895 #endif 896 897 static void ss_bad_rpl(struct ex_regs *regs) 898 { 899 extern char ss_bad_rpl_cont; 900 901 ++exceptions; 902 regs->rip = (ulong)&ss_bad_rpl_cont; 903 } 904 905 static void test_sreg(volatile uint16_t *mem) 906 { 907 u16 ss = read_ss(); 908 909 // check for null segment load 910 *mem = 0; 911 asm volatile("mov %0, %%ss" : : "m"(*mem)); 912 report("mov null, %%ss", read_ss() == 0); 913 914 // check for exception when ss.rpl != cpl on null segment load 915 exceptions = 0; 916 handle_exception(GP_VECTOR, ss_bad_rpl); 917 *mem = 3; 918 asm volatile("mov %0, %%ss; ss_bad_rpl_cont:" : : "m"(*mem)); 919 report("mov null, %%ss (with ss.rpl != cpl)", exceptions == 1 && read_ss() == 0); 920 handle_exception(GP_VECTOR, 0); 921 write_ss(ss); 922 } 923 924 /* Broken emulation causes triple fault, which skips the other tests. */ 925 #if 0 926 static void test_lldt(volatile uint16_t *mem) 927 { 928 u64 gdt[] = { 0, /* null descriptor */ 929 #ifdef __X86_64__ 930 0, /* ldt descriptor is 16 bytes in long mode */ 931 #endif 932 0x0000f82000000ffffull /* ldt descriptor */ }; 933 struct descriptor_table_ptr gdt_ptr = { .limit = sizeof(gdt) - 1, 934 .base = (ulong)&gdt }; 935 struct descriptor_table_ptr orig_gdt; 936 937 cli(); 938 sgdt(&orig_gdt); 939 lgdt(&gdt_ptr); 940 *mem = 0x8; 941 asm volatile("lldt %0" : : "m"(*mem)); 942 lgdt(&orig_gdt); 943 sti(); 944 report("lldt", sldt() == *mem); 945 } 946 #endif 947 948 static void test_ltr(volatile uint16_t *mem) 949 { 950 struct descriptor_table_ptr gdt_ptr; 951 uint64_t *gdt, *trp; 952 uint16_t tr = str(); 953 uint64_t busy_mask = (uint64_t)1 << 41; 954 955 sgdt(&gdt_ptr); 956 gdt = (uint64_t *)gdt_ptr.base; 957 trp = &gdt[tr >> 3]; 958 *trp &= ~busy_mask; 959 *mem = tr; 960 asm volatile("ltr %0" : : "m"(*mem) : "memory"); 961 report("ltr", str() == tr && (*trp & busy_mask)); 962 } 963 964 static void test_simplealu(u32 *mem) 965 { 966 *mem = 0x1234; 967 asm("or %1, %0" : "+m"(*mem) : "r"(0x8001)); 968 report("or", *mem == 0x9235); 969 asm("add %1, %0" : "+m"(*mem) : "r"(2)); 970 report("add", *mem == 0x9237); 971 asm("xor %1, %0" : "+m"(*mem) : "r"(0x1111)); 972 report("xor", *mem == 0x8326); 973 asm("sub %1, %0" : "+m"(*mem) : "r"(0x26)); 974 report("sub", *mem == 0x8300); 975 asm("clc; adc %1, %0" : "+m"(*mem) : "r"(0x100)); 976 report("adc(0)", *mem == 0x8400); 977 asm("stc; adc %1, %0" : "+m"(*mem) : "r"(0x100)); 978 report("adc(0)", *mem == 0x8501); 979 asm("clc; sbb %1, %0" : "+m"(*mem) : "r"(0)); 980 report("sbb(0)", *mem == 0x8501); 981 asm("stc; sbb %1, %0" : "+m"(*mem) : "r"(0)); 982 report("sbb(1)", *mem == 0x8500); 983 asm("and %1, %0" : "+m"(*mem) : "r"(0xfe77)); 984 report("and", *mem == 0x8400); 985 asm("test %1, %0" : "+m"(*mem) : "r"(0xf000)); 986 report("test", *mem == 0x8400); 987 } 988 989 static void illegal_movbe_handler(struct ex_regs *regs) 990 { 991 extern char bad_movbe_cont; 992 993 ++exceptions; 994 regs->rip = (ulong)&bad_movbe_cont; 995 } 996 997 static void test_illegal_movbe(void) 998 { 999 if (!this_cpu_has(X86_FEATURE_MOVBE)) { 1000 report_skip("illegal movbe"); 1001 return; 1002 } 1003 1004 exceptions = 0; 1005 handle_exception(UD_VECTOR, illegal_movbe_handler); 1006 asm volatile(".byte 0x0f; .byte 0x38; .byte 0xf0; .byte 0xc0;\n\t" 1007 " bad_movbe_cont:" : : : "rax"); 1008 report("illegal movbe", exceptions == 1); 1009 handle_exception(UD_VECTOR, 0); 1010 } 1011 1012 static void record_no_fep(struct ex_regs *regs) 1013 { 1014 fep_available = 0; 1015 regs->rip += KVM_FEP_LENGTH; 1016 } 1017 1018 int main(void) 1019 { 1020 void *mem; 1021 void *insn_page; 1022 void *insn_ram; 1023 unsigned long t1, t2; 1024 1025 setup_vm(); 1026 setup_idt(); 1027 handle_exception(UD_VECTOR, record_no_fep); 1028 asm(KVM_FEP "nop"); 1029 handle_exception(UD_VECTOR, 0); 1030 1031 mem = alloc_vpages(2); 1032 install_page((void *)read_cr3(), IORAM_BASE_PHYS, mem); 1033 // install the page twice to test cross-page mmio 1034 install_page((void *)read_cr3(), IORAM_BASE_PHYS, mem + 4096); 1035 insn_page = alloc_page(); 1036 insn_ram = vmap(virt_to_phys(insn_page), 4096); 1037 1038 // test mov reg, r/m and mov r/m, reg 1039 t1 = 0x123456789abcdef; 1040 asm volatile("mov %[t1], (%[mem]) \n\t" 1041 "mov (%[mem]), %[t2]" 1042 : [t2]"=r"(t2) 1043 : [t1]"r"(t1), [mem]"r"(mem) 1044 : "memory"); 1045 report("mov reg, r/m (1)", t2 == 0x123456789abcdef); 1046 1047 test_simplealu(mem); 1048 test_cmps(mem); 1049 test_scas(mem); 1050 1051 test_push(mem); 1052 test_pop(mem); 1053 1054 test_xchg(mem); 1055 test_xadd(mem); 1056 1057 test_cr8(); 1058 1059 test_smsw(mem); 1060 test_lmsw(); 1061 test_ljmp(mem); 1062 test_stringio(); 1063 test_incdecnotneg(mem); 1064 test_btc(mem); 1065 test_bsfbsr(mem); 1066 test_imul(mem); 1067 test_muldiv(mem); 1068 test_sse(mem); 1069 test_mmx(mem); 1070 test_rip_relative(mem, insn_ram); 1071 test_shld_shrd(mem); 1072 //test_lgdt_lidt(mem); 1073 test_sreg(mem); 1074 //test_lldt(mem); 1075 test_ltr(mem); 1076 test_cmov(mem); 1077 1078 if (fep_available) { 1079 test_mmx_movq_mf(mem); 1080 test_movabs(mem); 1081 test_smsw_reg(mem); 1082 test_nop(mem); 1083 test_mov_dr(mem); 1084 } else { 1085 report_skip("skipping register-only tests, " 1086 "use kvm.forced_emulation_prefix=1 to enable"); 1087 } 1088 1089 test_push16(mem); 1090 test_crosspage_mmio(mem); 1091 1092 test_string_io_mmio(mem); 1093 1094 test_jmp_noncanonical(mem); 1095 test_illegal_movbe(); 1096 1097 return report_summary(); 1098 } 1099