1 #include "ioram.h" 2 #include "vm.h" 3 #include "libcflat.h" 4 #include "desc.h" 5 #include "types.h" 6 7 #define memset __builtin_memset 8 #define TESTDEV_IO_PORT 0xe0 9 10 static int exceptions; 11 12 struct regs { 13 u64 rax, rbx, rcx, rdx; 14 u64 rsi, rdi, rsp, rbp; 15 u64 r8, r9, r10, r11; 16 u64 r12, r13, r14, r15; 17 u64 rip, rflags; 18 }; 19 struct regs inregs, outregs, save; 20 21 struct insn_desc { 22 u64 ptr; 23 size_t len; 24 }; 25 26 static char st1[] = "abcdefghijklmnop"; 27 28 void test_stringio() 29 { 30 unsigned char r = 0; 31 asm volatile("cld \n\t" 32 "movw %0, %%dx \n\t" 33 "rep outsb \n\t" 34 : : "i"((short)TESTDEV_IO_PORT), 35 "S"(st1), "c"(sizeof(st1) - 1)); 36 asm volatile("inb %1, %0\n\t" : "=a"(r) : "i"((short)TESTDEV_IO_PORT)); 37 report("outsb up", r == st1[sizeof(st1) - 2]); /* last char */ 38 39 asm volatile("std \n\t" 40 "movw %0, %%dx \n\t" 41 "rep outsb \n\t" 42 : : "i"((short)TESTDEV_IO_PORT), 43 "S"(st1 + sizeof(st1) - 2), "c"(sizeof(st1) - 1)); 44 asm volatile("cld \n\t" : : ); 45 asm volatile("in %1, %0\n\t" : "=a"(r) : "i"((short)TESTDEV_IO_PORT)); 46 report("outsb down", r == st1[0]); 47 } 48 49 void test_cmps_one(unsigned char *m1, unsigned char *m3) 50 { 51 void *rsi, *rdi; 52 long rcx, tmp; 53 54 rsi = m1; rdi = m3; rcx = 30; 55 asm volatile("xor %[tmp], %[tmp] \n\t" 56 "repe/cmpsb" 57 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 58 : : "cc"); 59 report("repe/cmpsb (1)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 60 61 rsi = m1; rdi = m3; rcx = 30; 62 asm volatile("or $1, %[tmp]\n\t" // clear ZF 63 "repe/cmpsb" 64 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 65 : : "cc"); 66 report("repe/cmpsb (1.zf)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 67 68 rsi = m1; rdi = m3; rcx = 15; 69 asm volatile("xor %[tmp], %[tmp] \n\t" 70 "repe/cmpsw" 71 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 72 : : "cc"); 73 report("repe/cmpsw (1)", rcx == 0 && rsi == m1 + 30 && rdi == m3 + 30); 74 75 rsi = m1; rdi = m3; rcx = 7; 76 asm volatile("xor %[tmp], %[tmp] \n\t" 77 "repe/cmpsl" 78 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 79 : : "cc"); 80 report("repe/cmpll (1)", rcx == 0 && rsi == m1 + 28 && rdi == m3 + 28); 81 82 rsi = m1; rdi = m3; rcx = 4; 83 asm volatile("xor %[tmp], %[tmp] \n\t" 84 "repe/cmpsq" 85 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 86 : : "cc"); 87 report("repe/cmpsq (1)", rcx == 0 && rsi == m1 + 32 && rdi == m3 + 32); 88 89 rsi = m1; rdi = m3; rcx = 130; 90 asm volatile("xor %[tmp], %[tmp] \n\t" 91 "repe/cmpsb" 92 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 93 : : "cc"); 94 report("repe/cmpsb (2)", 95 rcx == 29 && rsi == m1 + 101 && rdi == m3 + 101); 96 97 rsi = m1; rdi = m3; rcx = 65; 98 asm volatile("xor %[tmp], %[tmp] \n\t" 99 "repe/cmpsw" 100 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 101 : : "cc"); 102 report("repe/cmpsw (2)", 103 rcx == 14 && rsi == m1 + 102 && rdi == m3 + 102); 104 105 rsi = m1; rdi = m3; rcx = 32; 106 asm volatile("xor %[tmp], %[tmp] \n\t" 107 "repe/cmpsl" 108 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 109 : : "cc"); 110 report("repe/cmpll (2)", 111 rcx == 6 && rsi == m1 + 104 && rdi == m3 + 104); 112 113 rsi = m1; rdi = m3; rcx = 16; 114 asm volatile("xor %[tmp], %[tmp] \n\t" 115 "repe/cmpsq" 116 : "+S"(rsi), "+D"(rdi), "+c"(rcx), [tmp]"=&r"(tmp) 117 : : "cc"); 118 report("repe/cmpsq (2)", 119 rcx == 3 && rsi == m1 + 104 && rdi == m3 + 104); 120 121 } 122 123 void test_cmps(void *mem) 124 { 125 unsigned char *m1 = mem, *m2 = mem + 1024; 126 unsigned char m3[1024]; 127 128 for (int i = 0; i < 100; ++i) 129 m1[i] = m2[i] = m3[i] = i; 130 for (int i = 100; i < 200; ++i) 131 m1[i] = (m3[i] = m2[i] = i) + 1; 132 test_cmps_one(m1, m3); 133 test_cmps_one(m1, m2); 134 } 135 136 void test_scas(void *mem) 137 { 138 bool z; 139 void *di; 140 141 *(ulong *)mem = 0x77665544332211; 142 143 di = mem; 144 asm ("scasb; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff11)); 145 report("scasb match", di == mem + 1 && z); 146 147 di = mem; 148 asm ("scasb; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff54)); 149 report("scasb mismatch", di == mem + 1 && !z); 150 151 di = mem; 152 asm ("scasw; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff2211)); 153 report("scasw match", di == mem + 2 && z); 154 155 di = mem; 156 asm ("scasw; setz %0" : "=rm"(z), "+D"(di) : "a"(0xffdd11)); 157 report("scasw mismatch", di == mem + 2 && !z); 158 159 di = mem; 160 asm ("scasl; setz %0" : "=rm"(z), "+D"(di) : "a"(0xff44332211ul)); 161 report("scasd match", di == mem + 4 && z); 162 163 di = mem; 164 asm ("scasl; setz %0" : "=rm"(z), "+D"(di) : "a"(0x45332211)); 165 report("scasd mismatch", di == mem + 4 && !z); 166 167 di = mem; 168 asm ("scasq; setz %0" : "=rm"(z), "+D"(di) : "a"(0x77665544332211ul)); 169 report("scasq match", di == mem + 8 && z); 170 171 di = mem; 172 asm ("scasq; setz %0" : "=rm"(z), "+D"(di) : "a"(3)); 173 report("scasq mismatch", di == mem + 8 && !z); 174 } 175 176 void test_cr8(void) 177 { 178 unsigned long src, dst; 179 180 dst = 777; 181 src = 3; 182 asm volatile("mov %[src], %%cr8; mov %%cr8, %[dst]" 183 : [dst]"+r"(dst), [src]"+r"(src)); 184 report("mov %cr8", dst == 3 && src == 3); 185 } 186 187 void test_push(void *mem) 188 { 189 unsigned long tmp; 190 unsigned long *stack_top = mem + 4096; 191 unsigned long *new_stack_top; 192 unsigned long memw = 0x123456789abcdeful; 193 194 memset(mem, 0x55, (void *)stack_top - mem); 195 196 asm volatile("mov %%rsp, %[tmp] \n\t" 197 "mov %[stack_top], %%rsp \n\t" 198 "pushq $-7 \n\t" 199 "pushq %[reg] \n\t" 200 "pushq (%[mem]) \n\t" 201 "pushq $-7070707 \n\t" 202 "mov %%rsp, %[new_stack_top] \n\t" 203 "mov %[tmp], %%rsp" 204 : [tmp]"=&r"(tmp), [new_stack_top]"=r"(new_stack_top) 205 : [stack_top]"r"(stack_top), 206 [reg]"r"(-17l), [mem]"r"(&memw) 207 : "memory"); 208 209 report("push $imm8", stack_top[-1] == -7ul); 210 report("push %reg", stack_top[-2] == -17ul); 211 report("push mem", stack_top[-3] == 0x123456789abcdeful); 212 report("push $imm", stack_top[-4] == -7070707); 213 } 214 215 void test_pop(void *mem) 216 { 217 unsigned long tmp, tmp3, rsp, rbp; 218 unsigned long *stack_top = mem + 4096; 219 unsigned long memw = 0x123456789abcdeful; 220 static unsigned long tmp2; 221 222 memset(mem, 0x55, (void *)stack_top - mem); 223 224 asm volatile("pushq %[val] \n\t" 225 "popq (%[mem])" 226 : : [val]"m"(memw), [mem]"r"(mem) : "memory"); 227 report("pop mem", *(unsigned long *)mem == memw); 228 229 memw = 7 - memw; 230 asm volatile("mov %%rsp, %[tmp] \n\t" 231 "mov %[stack_top], %%rsp \n\t" 232 "pushq %[val] \n\t" 233 "popq %[tmp2] \n\t" 234 "mov %[tmp], %%rsp" 235 : [tmp]"=&r"(tmp), [tmp2]"=m"(tmp2) 236 : [val]"r"(memw), [stack_top]"r"(stack_top) 237 : "memory"); 238 report("pop mem (2)", tmp2 == memw); 239 240 memw = 129443 - memw; 241 asm volatile("mov %%rsp, %[tmp] \n\t" 242 "mov %[stack_top], %%rsp \n\t" 243 "pushq %[val] \n\t" 244 "popq %[tmp2] \n\t" 245 "mov %[tmp], %%rsp" 246 : [tmp]"=&r"(tmp), [tmp2]"=r"(tmp2) 247 : [val]"r"(memw), [stack_top]"r"(stack_top) 248 : "memory"); 249 report("pop reg", tmp2 == memw); 250 251 asm volatile("mov %%rsp, %[tmp] \n\t" 252 "mov %[stack_top], %%rsp \n\t" 253 "push $1f \n\t" 254 "ret \n\t" 255 "2: jmp 2b \n\t" 256 "1: mov %[tmp], %%rsp" 257 : [tmp]"=&r"(tmp) : [stack_top]"r"(stack_top) 258 : "memory"); 259 report("ret", 1); 260 261 stack_top[-1] = 0x778899; 262 asm volatile("mov %%rsp, %[tmp] \n\t" 263 "mov %%rbp, %[tmp3] \n\t" 264 "mov %[stack_top], %%rbp \n\t" 265 "leave \n\t" 266 "xchg %%rsp, %[tmp] \n\t" 267 "xchg %%rbp, %[tmp3]" 268 : [tmp]"=&r"(tmp), [tmp3]"=&r"(tmp3) : [stack_top]"r"(stack_top-1) 269 : "memory"); 270 report("leave", tmp == (ulong)stack_top && tmp3 == 0x778899); 271 272 rbp = 0xaa55aa55bb66bb66ULL; 273 rsp = (unsigned long)stack_top; 274 asm volatile("xchg %%rsp, %[rsp] \n\t" 275 "xchg %%rbp, %[rbp] \n\t" 276 "enter $0x1238, $0 \n\t" 277 "xchg %%rsp, %[rsp] \n\t" 278 "xchg %%rbp, %[rbp]" 279 : [rsp]"+a"(rsp), [rbp]"+b"(rbp) : : "memory"); 280 report("enter", 281 rsp == (unsigned long)stack_top - 8 - 0x1238 282 && rbp == (unsigned long)stack_top - 8 283 && stack_top[-1] == 0xaa55aa55bb66bb66ULL); 284 } 285 286 void test_ljmp(void *mem) 287 { 288 unsigned char *m = mem; 289 volatile int res = 1; 290 291 *(unsigned long**)m = &&jmpf; 292 asm volatile ("data16/mov %%cs, %0":"=m"(*(m + sizeof(unsigned long)))); 293 asm volatile ("rex64/ljmp *%0"::"m"(*m)); 294 res = 0; 295 jmpf: 296 report("ljmp", res); 297 } 298 299 void test_incdecnotneg(void *mem) 300 { 301 unsigned long *m = mem, v = 1234; 302 unsigned char *mb = mem, vb = 66; 303 304 *m = 0; 305 306 asm volatile ("incl %0":"+m"(*m)); 307 report("incl", *m == 1); 308 asm volatile ("decl %0":"+m"(*m)); 309 report("decl", *m == 0); 310 asm volatile ("incb %0":"+m"(*m)); 311 report("incb", *m == 1); 312 asm volatile ("decb %0":"+m"(*m)); 313 report("decb", *m == 0); 314 315 asm volatile ("lock incl %0":"+m"(*m)); 316 report("lock incl", *m == 1); 317 asm volatile ("lock decl %0":"+m"(*m)); 318 report("lock decl", *m == 0); 319 asm volatile ("lock incb %0":"+m"(*m)); 320 report("lock incb", *m == 1); 321 asm volatile ("lock decb %0":"+m"(*m)); 322 report("lock decb", *m == 0); 323 324 *m = v; 325 326 asm ("lock negq %0" : "+m"(*m)); v = -v; 327 report("lock negl", *m == v); 328 asm ("lock notq %0" : "+m"(*m)); v = ~v; 329 report("lock notl", *m == v); 330 331 *mb = vb; 332 333 asm ("lock negb %0" : "+m"(*mb)); vb = -vb; 334 report("lock negb", *mb == vb); 335 asm ("lock notb %0" : "+m"(*mb)); vb = ~vb; 336 report("lock notb", *mb == vb); 337 } 338 339 void test_smsw(uint64_t *h_mem) 340 { 341 char mem[16]; 342 unsigned short msw, msw_orig, *pmsw; 343 int i, zero; 344 345 msw_orig = read_cr0(); 346 347 asm("smsw %0" : "=r"(msw)); 348 report("smsw (1)", msw == msw_orig); 349 350 memset(mem, 0, 16); 351 pmsw = (void *)mem; 352 asm("smsw %0" : "=m"(pmsw[4])); 353 zero = 1; 354 for (i = 0; i < 8; ++i) 355 if (i != 4 && pmsw[i]) 356 zero = 0; 357 report("smsw (2)", msw == pmsw[4] && zero); 358 359 /* Trigger exit on smsw */ 360 *h_mem = 0x12345678abcdeful; 361 asm volatile("smsw %0" : "=m"(*h_mem)); 362 report("smsw (3)", msw == (unsigned short)*h_mem && 363 (*h_mem & ~0xfffful) == 0x12345678ab0000ul); 364 } 365 366 void test_lmsw(void) 367 { 368 char mem[16]; 369 unsigned short msw, *pmsw; 370 unsigned long cr0; 371 372 cr0 = read_cr0(); 373 374 msw = cr0 ^ 8; 375 asm("lmsw %0" : : "r"(msw)); 376 printf("before %lx after %lx\n", cr0, read_cr0()); 377 report("lmsw (1)", (cr0 ^ read_cr0()) == 8); 378 379 pmsw = (void *)mem; 380 *pmsw = cr0; 381 asm("lmsw %0" : : "m"(*pmsw)); 382 printf("before %lx after %lx\n", cr0, read_cr0()); 383 report("lmsw (2)", cr0 == read_cr0()); 384 385 /* lmsw can't clear cr0.pe */ 386 msw = (cr0 & ~1ul) ^ 4; /* change EM to force trap */ 387 asm("lmsw %0" : : "r"(msw)); 388 report("lmsw (3)", (cr0 ^ read_cr0()) == 4 && (cr0 & 1)); 389 390 /* back to normal */ 391 msw = cr0; 392 asm("lmsw %0" : : "r"(msw)); 393 } 394 395 void test_xchg(void *mem) 396 { 397 unsigned long *memq = mem; 398 unsigned long rax; 399 400 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 401 "mov %%rax, (%[memq])\n\t" 402 "mov $0xfedcba9876543210, %%rax\n\t" 403 "xchg %%al, (%[memq])\n\t" 404 "mov %%rax, %[rax]\n\t" 405 : [rax]"=r"(rax) 406 : [memq]"r"(memq) 407 : "memory"); 408 report("xchg reg, r/m (1)", 409 rax == 0xfedcba98765432ef && *memq == 0x123456789abcd10); 410 411 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 412 "mov %%rax, (%[memq])\n\t" 413 "mov $0xfedcba9876543210, %%rax\n\t" 414 "xchg %%ax, (%[memq])\n\t" 415 "mov %%rax, %[rax]\n\t" 416 : [rax]"=r"(rax) 417 : [memq]"r"(memq) 418 : "memory"); 419 report("xchg reg, r/m (2)", 420 rax == 0xfedcba987654cdef && *memq == 0x123456789ab3210); 421 422 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 423 "mov %%rax, (%[memq])\n\t" 424 "mov $0xfedcba9876543210, %%rax\n\t" 425 "xchg %%eax, (%[memq])\n\t" 426 "mov %%rax, %[rax]\n\t" 427 : [rax]"=r"(rax) 428 : [memq]"r"(memq) 429 : "memory"); 430 report("xchg reg, r/m (3)", 431 rax == 0x89abcdef && *memq == 0x123456776543210); 432 433 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 434 "mov %%rax, (%[memq])\n\t" 435 "mov $0xfedcba9876543210, %%rax\n\t" 436 "xchg %%rax, (%[memq])\n\t" 437 "mov %%rax, %[rax]\n\t" 438 : [rax]"=r"(rax) 439 : [memq]"r"(memq) 440 : "memory"); 441 report("xchg reg, r/m (4)", 442 rax == 0x123456789abcdef && *memq == 0xfedcba9876543210); 443 } 444 445 void test_xadd(void *mem) 446 { 447 unsigned long *memq = mem; 448 unsigned long rax; 449 450 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 451 "mov %%rax, (%[memq])\n\t" 452 "mov $0xfedcba9876543210, %%rax\n\t" 453 "xadd %%al, (%[memq])\n\t" 454 "mov %%rax, %[rax]\n\t" 455 : [rax]"=r"(rax) 456 : [memq]"r"(memq) 457 : "memory"); 458 report("xadd reg, r/m (1)", 459 rax == 0xfedcba98765432ef && *memq == 0x123456789abcdff); 460 461 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 462 "mov %%rax, (%[memq])\n\t" 463 "mov $0xfedcba9876543210, %%rax\n\t" 464 "xadd %%ax, (%[memq])\n\t" 465 "mov %%rax, %[rax]\n\t" 466 : [rax]"=r"(rax) 467 : [memq]"r"(memq) 468 : "memory"); 469 report("xadd reg, r/m (2)", 470 rax == 0xfedcba987654cdef && *memq == 0x123456789abffff); 471 472 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 473 "mov %%rax, (%[memq])\n\t" 474 "mov $0xfedcba9876543210, %%rax\n\t" 475 "xadd %%eax, (%[memq])\n\t" 476 "mov %%rax, %[rax]\n\t" 477 : [rax]"=r"(rax) 478 : [memq]"r"(memq) 479 : "memory"); 480 report("xadd reg, r/m (3)", 481 rax == 0x89abcdef && *memq == 0x1234567ffffffff); 482 483 asm volatile("mov $0x123456789abcdef, %%rax\n\t" 484 "mov %%rax, (%[memq])\n\t" 485 "mov $0xfedcba9876543210, %%rax\n\t" 486 "xadd %%rax, (%[memq])\n\t" 487 "mov %%rax, %[rax]\n\t" 488 : [rax]"=r"(rax) 489 : [memq]"r"(memq) 490 : "memory"); 491 report("xadd reg, r/m (4)", 492 rax == 0x123456789abcdef && *memq == 0xffffffffffffffff); 493 } 494 495 void test_btc(void *mem) 496 { 497 unsigned int *a = mem; 498 499 memset(mem, 0, 4 * sizeof(unsigned int)); 500 501 asm ("btcl $32, %0" :: "m"(a[0]) : "memory"); 502 asm ("btcl $1, %0" :: "m"(a[1]) : "memory"); 503 asm ("btcl %1, %0" :: "m"(a[0]), "r"(66) : "memory"); 504 report("btcl imm8, r/m", a[0] == 1 && a[1] == 2 && a[2] == 4); 505 506 asm ("btcl %1, %0" :: "m"(a[3]), "r"(-1) : "memory"); 507 report("btcl reg, r/m", a[0] == 1 && a[1] == 2 && a[2] == 0x80000004); 508 509 asm ("btcq %1, %0" : : "m"(a[2]), "r"(-1l) : "memory"); 510 report("btcq reg, r/m", a[0] == 1 && a[1] == 0x80000002 && 511 a[2] == 0x80000004 && a[3] == 0); 512 } 513 514 void test_bsfbsr(void *mem) 515 { 516 unsigned long rax, *memq = mem; 517 unsigned eax, *meml = mem; 518 unsigned short ax, *memw = mem; 519 unsigned char z; 520 521 *memw = 0xc000; 522 asm("bsfw %[mem], %[a]" : [a]"=a"(ax) : [mem]"m"(*memw)); 523 report("bsfw r/m, reg", ax == 14); 524 525 *meml = 0xc0000000; 526 asm("bsfl %[mem], %[a]" : [a]"=a"(eax) : [mem]"m"(*meml)); 527 report("bsfl r/m, reg", eax == 30); 528 529 *memq = 0xc00000000000; 530 asm("bsfq %[mem], %[a]" : [a]"=a"(rax) : [mem]"m"(*memq)); 531 report("bsfq r/m, reg", rax == 46); 532 533 *memq = 0; 534 asm("bsfq %[mem], %[a]; setz %[z]" 535 : [a]"=a"(rax), [z]"=rm"(z) : [mem]"m"(*memq)); 536 report("bsfq r/m, reg", z == 1); 537 538 *memw = 0xc000; 539 asm("bsrw %[mem], %[a]" : [a]"=a"(ax) : [mem]"m"(*memw)); 540 report("bsrw r/m, reg", ax == 15); 541 542 *meml = 0xc0000000; 543 asm("bsrl %[mem], %[a]" : [a]"=a"(eax) : [mem]"m"(*meml)); 544 report("bsrl r/m, reg", eax == 31); 545 546 *memq = 0xc00000000000; 547 asm("bsrq %[mem], %[a]" : [a]"=a"(rax) : [mem]"m"(*memq)); 548 report("bsrq r/m, reg", rax == 47); 549 550 *memq = 0; 551 asm("bsrq %[mem], %[a]; setz %[z]" 552 : [a]"=a"(rax), [z]"=rm"(z) : [mem]"m"(*memq)); 553 report("bsrq r/m, reg", z == 1); 554 } 555 556 static void test_imul(ulong *mem) 557 { 558 ulong a; 559 560 *mem = 51; a = 0x1234567812345678UL; 561 asm ("imulw %1, %%ax" : "+a"(a) : "m"(*mem)); 562 report("imul ax, mem", a == 0x12345678123439e8); 563 564 *mem = 51; a = 0x1234567812345678UL; 565 asm ("imull %1, %%eax" : "+a"(a) : "m"(*mem)); 566 report("imul eax, mem", a == 0xa06d39e8); 567 568 *mem = 51; a = 0x1234567812345678UL; 569 asm ("imulq %1, %%rax" : "+a"(a) : "m"(*mem)); 570 report("imul rax, mem", a == 0xA06D39EBA06D39E8UL); 571 572 *mem = 0x1234567812345678UL; a = 0x8765432187654321L; 573 asm ("imulw $51, %1, %%ax" : "+a"(a) : "m"(*mem)); 574 report("imul ax, mem, imm8", a == 0x87654321876539e8); 575 576 *mem = 0x1234567812345678UL; 577 asm ("imull $51, %1, %%eax" : "+a"(a) : "m"(*mem)); 578 report("imul eax, mem, imm8", a == 0xa06d39e8); 579 580 *mem = 0x1234567812345678UL; 581 asm ("imulq $51, %1, %%rax" : "+a"(a) : "m"(*mem)); 582 report("imul rax, mem, imm8", a == 0xA06D39EBA06D39E8UL); 583 584 *mem = 0x1234567812345678UL; a = 0x8765432187654321L; 585 asm ("imulw $311, %1, %%ax" : "+a"(a) : "m"(*mem)); 586 report("imul ax, mem, imm", a == 0x8765432187650bc8); 587 588 *mem = 0x1234567812345678UL; 589 asm ("imull $311, %1, %%eax" : "+a"(a) : "m"(*mem)); 590 report("imul eax, mem, imm", a == 0x1d950bc8); 591 592 *mem = 0x1234567812345678UL; 593 asm ("imulq $311, %1, %%rax" : "+a"(a) : "m"(*mem)); 594 report("imul rax, mem, imm", a == 0x1D950BDE1D950BC8L); 595 } 596 597 static void test_muldiv(long *mem) 598 { 599 long a, d, aa, dd; 600 u8 ex = 1; 601 602 *mem = 0; a = 1; d = 2; 603 asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:" 604 : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem)); 605 report("divq (fault)", a == 1 && d == 2 && ex); 606 607 *mem = 987654321098765UL; a = 123456789012345UL; d = 123456789012345UL; 608 asm (ASM_TRY("1f") "divq %3; movb $0, %2; 1:" 609 : "+a"(a), "+d"(d), "+q"(ex) : "m"(*mem)); 610 report("divq (1)", 611 a == 0x1ffffffb1b963b33ul && d == 0x273ba4384ede2ul && !ex); 612 aa = 0x1111111111111111; dd = 0x2222222222222222; 613 *mem = 0x3333333333333333; a = aa; d = dd; 614 asm("mulb %2" : "+a"(a), "+d"(d) : "m"(*mem)); 615 report("mulb mem", a == 0x1111111111110363 && d == dd); 616 *mem = 0x3333333333333333; a = aa; d = dd; 617 asm("mulw %2" : "+a"(a), "+d"(d) : "m"(*mem)); 618 report("mulw mem", a == 0x111111111111c963 && d == 0x2222222222220369); 619 *mem = 0x3333333333333333; a = aa; d = dd; 620 asm("mull %2" : "+a"(a), "+d"(d) : "m"(*mem)); 621 report("mull mem", a == 0x962fc963 && d == 0x369d036); 622 *mem = 0x3333333333333333; a = aa; d = dd; 623 asm("mulq %2" : "+a"(a), "+d"(d) : "m"(*mem)); 624 report("mulq mem", a == 0x2fc962fc962fc963 && d == 0x369d0369d0369d0); 625 } 626 627 typedef unsigned __attribute__((vector_size(16))) sse128; 628 629 typedef union { 630 sse128 sse; 631 unsigned u[4]; 632 } sse_union; 633 634 static bool sseeq(sse_union *v1, sse_union *v2) 635 { 636 bool ok = true; 637 int i; 638 639 for (i = 0; i < 4; ++i) { 640 ok &= v1->u[i] == v2->u[i]; 641 } 642 643 return ok; 644 } 645 646 static void test_sse(sse_union *mem) 647 { 648 sse_union v; 649 650 write_cr0(read_cr0() & ~6); /* EM, TS */ 651 write_cr4(read_cr4() | 0x200); /* OSFXSR */ 652 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 653 asm("movdqu %1, %0" : "=m"(*mem) : "x"(v.sse)); 654 report("movdqu (read)", sseeq(&v, mem)); 655 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 656 asm("movdqu %1, %0" : "=x"(v.sse) : "m"(*mem)); 657 report("movdqu (write)", sseeq(mem, &v)); 658 659 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 660 asm("movaps %1, %0" : "=m"(*mem) : "x"(v.sse)); 661 report("movaps (read)", sseeq(mem, &v)); 662 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 663 asm("movaps %1, %0" : "=x"(v.sse) : "m"(*mem)); 664 report("movaps (write)", sseeq(&v, mem)); 665 666 v.u[0] = 1; v.u[1] = 2; v.u[2] = 3; v.u[3] = 4; 667 asm("movapd %1, %0" : "=m"(*mem) : "x"(v.sse)); 668 report("movapd (read)", sseeq(mem, &v)); 669 mem->u[0] = 5; mem->u[1] = 6; mem->u[2] = 7; mem->u[3] = 8; 670 asm("movapd %1, %0" : "=x"(v.sse) : "m"(*mem)); 671 report("movapd (write)", sseeq(&v, mem)); 672 } 673 674 static void test_mmx(uint64_t *mem) 675 { 676 uint64_t v; 677 678 write_cr0(read_cr0() & ~6); /* EM, TS */ 679 asm volatile("fninit"); 680 v = 0x0102030405060708ULL; 681 asm("movq %1, %0" : "=m"(*mem) : "y"(v)); 682 report("movq (mmx, read)", v == *mem); 683 *mem = 0x8070605040302010ull; 684 asm("movq %1, %0" : "=y"(v) : "m"(*mem)); 685 report("movq (mmx, write)", v == *mem); 686 } 687 688 static void test_rip_relative(unsigned *mem, char *insn_ram) 689 { 690 /* movb $1, mem+2(%rip) */ 691 insn_ram[0] = 0xc6; 692 insn_ram[1] = 0x05; 693 *(unsigned *)&insn_ram[2] = 2 + (char *)mem - (insn_ram + 7); 694 insn_ram[6] = 0x01; 695 /* ret */ 696 insn_ram[7] = 0xc3; 697 698 *mem = 0; 699 asm("callq *%1" : "+m"(*mem) : "r"(insn_ram)); 700 report("movb $imm, 0(%rip)", *mem == 0x10000); 701 } 702 703 static void test_shld_shrd(u32 *mem) 704 { 705 *mem = 0x12345678; 706 asm("shld %2, %1, %0" : "+m"(*mem) : "r"(0xaaaaaaaaU), "c"((u8)3)); 707 report("shld (cl)", *mem == ((0x12345678 << 3) | 5)); 708 *mem = 0x12345678; 709 asm("shrd %2, %1, %0" : "+m"(*mem) : "r"(0x55555555U), "c"((u8)3)); 710 report("shrd (cl)", *mem == ((0x12345678 >> 3) | (5u << 29))); 711 } 712 713 static void test_cmov(u32 *mem) 714 { 715 u64 val; 716 *mem = 0xabcdef12u; 717 asm ("movq $0x1234567812345678, %%rax\n\t" 718 "cmpl %%eax, %%eax\n\t" 719 "cmovnel (%[mem]), %%eax\n\t" 720 "movq %%rax, %[val]\n\t" 721 : [val]"=r"(val) : [mem]"r"(mem) : "%rax", "cc"); 722 report("cmovnel", val == 0x12345678ul); 723 } 724 725 #define INSN_XCHG_ALL \ 726 "xchg %rax, 0+save \n\t" \ 727 "xchg %rbx, 8+save \n\t" \ 728 "xchg %rcx, 16+save \n\t" \ 729 "xchg %rdx, 24+save \n\t" \ 730 "xchg %rsi, 32+save \n\t" \ 731 "xchg %rdi, 40+save \n\t" \ 732 "xchg %rsp, 48+save \n\t" \ 733 "xchg %rbp, 56+save \n\t" \ 734 "xchg %r8, 64+save \n\t" \ 735 "xchg %r9, 72+save \n\t" \ 736 "xchg %r10, 80+save \n\t" \ 737 "xchg %r11, 88+save \n\t" \ 738 "xchg %r12, 96+save \n\t" \ 739 "xchg %r13, 104+save \n\t" \ 740 "xchg %r14, 112+save \n\t" \ 741 "xchg %r15, 120+save \n\t" 742 743 asm( 744 ".align 4096\n\t" 745 "insn_page:\n\t" 746 "ret\n\t" 747 "pushf\n\t" 748 "push 136+save \n\t" 749 "popf \n\t" 750 INSN_XCHG_ALL 751 "test_insn:\n\t" 752 "in (%dx),%al\n\t" 753 ".skip 31, 0x90\n\t" 754 "test_insn_end:\n\t" 755 INSN_XCHG_ALL 756 "pushf \n\t" 757 "pop 136+save \n\t" 758 "popf \n\t" 759 "ret \n\t" 760 "insn_page_end:\n\t" 761 ".align 4096\n\t" 762 ); 763 764 #define MK_INSN(name, str) \ 765 asm ( \ 766 ".pushsection .data.insn \n\t" \ 767 "insn_" #name ": \n\t" \ 768 ".quad 1001f, 1002f - 1001f \n\t" \ 769 ".popsection \n\t" \ 770 ".pushsection .text.insn, \"ax\" \n\t" \ 771 "1001: \n\t" \ 772 "insn_code_" #name ": " str " \n\t" \ 773 "1002: \n\t" \ 774 ".popsection" \ 775 ); \ 776 extern struct insn_desc insn_##name; 777 778 static void trap_emulator(uint64_t *mem, void *alt_insn_page, 779 struct insn_desc *alt_insn) 780 { 781 ulong *cr3 = (ulong *)read_cr3(); 782 void *insn_ram; 783 extern u8 insn_page[], test_insn[]; 784 785 insn_ram = vmap(virt_to_phys(insn_page), 4096); 786 memcpy(alt_insn_page, insn_page, 4096); 787 memcpy(alt_insn_page + (test_insn - insn_page), 788 (void *)(alt_insn->ptr), alt_insn->len); 789 save = inregs; 790 791 /* Load the code TLB with insn_page, but point the page tables at 792 alt_insn_page (and keep the data TLB clear, for AMD decode assist). 793 This will make the CPU trap on the insn_page instruction but the 794 hypervisor will see alt_insn_page. */ 795 install_page(cr3, virt_to_phys(insn_page), insn_ram); 796 invlpg(insn_ram); 797 /* Load code TLB */ 798 asm volatile("call *%0" : : "r"(insn_ram)); 799 install_page(cr3, virt_to_phys(alt_insn_page), insn_ram); 800 /* Trap, let hypervisor emulate at alt_insn_page */ 801 asm volatile("call *%0": : "r"(insn_ram+1)); 802 803 outregs = save; 804 } 805 806 static void advance_rip_by_3_and_note_exception(struct ex_regs *regs) 807 { 808 ++exceptions; 809 regs->rip += 3; 810 } 811 812 static void test_mmx_movq_mf(uint64_t *mem, uint8_t *insn_page, 813 uint8_t *alt_insn_page, void *insn_ram) 814 { 815 uint16_t fcw = 0; /* all exceptions unmasked */ 816 /* movq %mm0, (%rax) */ 817 void *stack = alloc_page(); 818 819 write_cr0(read_cr0() & ~6); /* TS, EM */ 820 exceptions = 0; 821 handle_exception(MF_VECTOR, advance_rip_by_3_and_note_exception); 822 asm volatile("fninit; fldcw %0" : : "m"(fcw)); 823 asm volatile("fldz; fldz; fdivp"); /* generate exception */ 824 825 MK_INSN(mmx_movq_mf, "movq %mm0, (%rax) \n\t"); 826 inregs = (struct regs){ .rsp=(u64)stack+1024 }; 827 trap_emulator(mem, alt_insn_page, &insn_mmx_movq_mf); 828 /* exit MMX mode */ 829 asm volatile("fnclex; emms"); 830 report("movq mmx generates #MF", exceptions == 1); 831 handle_exception(MF_VECTOR, 0); 832 } 833 834 static void test_movabs(uint64_t *mem, uint8_t *insn_page, 835 uint8_t *alt_insn_page, void *insn_ram) 836 { 837 /* mov $0x9090909090909090, %rcx */ 838 MK_INSN(movabs, "mov $0x9090909090909090, %rcx\n\t"); 839 inregs = (struct regs){ 0 }; 840 trap_emulator(mem, alt_insn_page, &insn_movabs); 841 report("64-bit mov imm2", outregs.rcx == 0x9090909090909090); 842 } 843 844 static void test_smsw_reg(uint64_t *mem, uint8_t *insn_page, 845 uint8_t *alt_insn_page, void *insn_ram) 846 { 847 unsigned long cr0 = read_cr0(); 848 inregs = (struct regs){ .rax = 0x1234567890abcdeful }; 849 850 MK_INSN(smsww, "smsww %ax\n\t"); 851 trap_emulator(mem, alt_insn_page, &insn_smsww); 852 report("16-bit smsw reg", (u16)outregs.rax == (u16)cr0 && 853 outregs.rax >> 16 == inregs.rax >> 16); 854 855 MK_INSN(smswl, "smswl %eax\n\t"); 856 trap_emulator(mem, alt_insn_page, &insn_smswl); 857 report("32-bit smsw reg", outregs.rax == (u32)cr0); 858 859 MK_INSN(smswq, "smswq %rax\n\t"); 860 trap_emulator(mem, alt_insn_page, &insn_smswq); 861 report("64-bit smsw reg", outregs.rax == cr0); 862 } 863 864 static void test_nop(uint64_t *mem, uint8_t *insn_page, 865 uint8_t *alt_insn_page, void *insn_ram) 866 { 867 inregs = (struct regs){ .rax = 0x1234567890abcdeful }; 868 MK_INSN(nop, "nop\n\t"); 869 trap_emulator(mem, alt_insn_page, &insn_nop); 870 report("nop", outregs.rax == inregs.rax); 871 } 872 873 static void test_crosspage_mmio(volatile uint8_t *mem) 874 { 875 volatile uint16_t w, *pw; 876 877 pw = (volatile uint16_t *)&mem[4095]; 878 mem[4095] = 0x99; 879 mem[4096] = 0x77; 880 asm volatile("mov %1, %0" : "=r"(w) : "m"(*pw) : "memory"); 881 report("cross-page mmio read", w == 0x7799); 882 asm volatile("mov %1, %0" : "=m"(*pw) : "r"((uint16_t)0x88aa)); 883 report("cross-page mmio write", mem[4095] == 0xaa && mem[4096] == 0x88); 884 } 885 886 static void test_string_io_mmio(volatile uint8_t *mem) 887 { 888 /* Cross MMIO pages.*/ 889 volatile uint8_t *mmio = mem + 4032; 890 891 asm volatile("outw %%ax, %%dx \n\t" : : "a"(0x9999), "d"(TESTDEV_IO_PORT)); 892 893 asm volatile ("cld; rep insb" : : "d" (TESTDEV_IO_PORT), "D" (mmio), "c" (1024)); 894 895 report("string_io_mmio", mmio[1023] == 0x99); 896 } 897 898 /* kvm doesn't allow lidt/lgdt from mmio, so the test is disabled */ 899 #if 0 900 static void test_lgdt_lidt(volatile uint8_t *mem) 901 { 902 struct descriptor_table_ptr orig, fresh = {}; 903 904 sgdt(&orig); 905 *(struct descriptor_table_ptr *)mem = (struct descriptor_table_ptr) { 906 .limit = 0xf234, 907 .base = 0x12345678abcd, 908 }; 909 cli(); 910 asm volatile("lgdt %0" : : "m"(*(struct descriptor_table_ptr *)mem)); 911 sgdt(&fresh); 912 lgdt(&orig); 913 sti(); 914 report("lgdt (long address)", orig.limit == fresh.limit && orig.base == fresh.base); 915 916 sidt(&orig); 917 *(struct descriptor_table_ptr *)mem = (struct descriptor_table_ptr) { 918 .limit = 0x432f, 919 .base = 0xdbca87654321, 920 }; 921 cli(); 922 asm volatile("lidt %0" : : "m"(*(struct descriptor_table_ptr *)mem)); 923 sidt(&fresh); 924 lidt(&orig); 925 sti(); 926 report("lidt (long address)", orig.limit == fresh.limit && orig.base == fresh.base); 927 } 928 #endif 929 930 static void ss_bad_rpl(struct ex_regs *regs) 931 { 932 extern char ss_bad_rpl_cont; 933 934 ++exceptions; 935 regs->rip = (ulong)&ss_bad_rpl_cont; 936 } 937 938 static void test_sreg(volatile uint16_t *mem) 939 { 940 u16 ss = read_ss(); 941 942 // check for null segment load 943 *mem = 0; 944 asm volatile("mov %0, %%ss" : : "m"(*mem)); 945 report("mov null, %%ss", read_ss() == 0); 946 947 // check for exception when ss.rpl != cpl on null segment load 948 exceptions = 0; 949 handle_exception(GP_VECTOR, ss_bad_rpl); 950 *mem = 3; 951 asm volatile("mov %0, %%ss; ss_bad_rpl_cont:" : : "m"(*mem)); 952 report("mov null, %%ss (with ss.rpl != cpl)", exceptions == 1 && read_ss() == 0); 953 handle_exception(GP_VECTOR, 0); 954 write_ss(ss); 955 } 956 957 static void test_lldt(volatile uint16_t *mem) 958 { 959 u64 gdt[] = { 0, 0x0000f82000000ffffull /* ldt descriptor */ }; 960 struct descriptor_table_ptr gdt_ptr = { .limit = 0xffff, .base = (ulong)&gdt }; 961 struct descriptor_table_ptr orig_gdt; 962 963 cli(); 964 sgdt(&orig_gdt); 965 lgdt(&gdt_ptr); 966 *mem = 0x8; 967 asm volatile("lldt %0" : : "m"(*mem)); 968 lgdt(&orig_gdt); 969 sti(); 970 report("lldt", sldt() == *mem); 971 } 972 973 static void test_ltr(volatile uint16_t *mem) 974 { 975 struct descriptor_table_ptr gdt_ptr; 976 uint64_t *gdt, *trp; 977 uint16_t tr = str(); 978 uint64_t busy_mask = (uint64_t)1 << 41; 979 980 sgdt(&gdt_ptr); 981 gdt = (uint64_t *)gdt_ptr.base; 982 trp = &gdt[tr >> 3]; 983 *trp &= ~busy_mask; 984 *mem = tr; 985 asm volatile("ltr %0" : : "m"(*mem) : "memory"); 986 report("ltr", str() == tr && (*trp & busy_mask)); 987 } 988 989 static void test_simplealu(u32 *mem) 990 { 991 *mem = 0x1234; 992 asm("or %1, %0" : "+m"(*mem) : "r"(0x8001)); 993 report("or", *mem == 0x9235); 994 asm("add %1, %0" : "+m"(*mem) : "r"(2)); 995 report("add", *mem == 0x9237); 996 asm("xor %1, %0" : "+m"(*mem) : "r"(0x1111)); 997 report("xor", *mem == 0x8326); 998 asm("sub %1, %0" : "+m"(*mem) : "r"(0x26)); 999 report("sub", *mem == 0x8300); 1000 asm("clc; adc %1, %0" : "+m"(*mem) : "r"(0x100)); 1001 report("adc(0)", *mem == 0x8400); 1002 asm("stc; adc %1, %0" : "+m"(*mem) : "r"(0x100)); 1003 report("adc(0)", *mem == 0x8501); 1004 asm("clc; sbb %1, %0" : "+m"(*mem) : "r"(0)); 1005 report("sbb(0)", *mem == 0x8501); 1006 asm("stc; sbb %1, %0" : "+m"(*mem) : "r"(0)); 1007 report("sbb(1)", *mem == 0x8500); 1008 asm("and %1, %0" : "+m"(*mem) : "r"(0xfe77)); 1009 report("and", *mem == 0x8400); 1010 asm("test %1, %0" : "+m"(*mem) : "r"(0xf000)); 1011 report("test", *mem == 0x8400); 1012 } 1013 1014 int main() 1015 { 1016 void *mem; 1017 void *insn_page, *alt_insn_page; 1018 void *insn_ram; 1019 unsigned long t1, t2; 1020 1021 setup_vm(); 1022 setup_idt(); 1023 mem = alloc_vpages(2); 1024 install_page((void *)read_cr3(), IORAM_BASE_PHYS, mem); 1025 // install the page twice to test cross-page mmio 1026 install_page((void *)read_cr3(), IORAM_BASE_PHYS, mem + 4096); 1027 insn_page = alloc_page(); 1028 alt_insn_page = alloc_page(); 1029 insn_ram = vmap(virt_to_phys(insn_page), 4096); 1030 1031 // test mov reg, r/m and mov r/m, reg 1032 t1 = 0x123456789abcdef; 1033 asm volatile("mov %[t1], (%[mem]) \n\t" 1034 "mov (%[mem]), %[t2]" 1035 : [t2]"=r"(t2) 1036 : [t1]"r"(t1), [mem]"r"(mem) 1037 : "memory"); 1038 report("mov reg, r/m (1)", t2 == 0x123456789abcdef); 1039 1040 test_simplealu(mem); 1041 test_cmps(mem); 1042 test_scas(mem); 1043 1044 test_push(mem); 1045 test_pop(mem); 1046 1047 test_xchg(mem); 1048 test_xadd(mem); 1049 1050 test_cr8(); 1051 1052 test_smsw(mem); 1053 test_lmsw(); 1054 test_ljmp(mem); 1055 test_stringio(); 1056 test_incdecnotneg(mem); 1057 test_btc(mem); 1058 test_bsfbsr(mem); 1059 test_imul(mem); 1060 test_muldiv(mem); 1061 test_sse(mem); 1062 test_mmx(mem); 1063 test_rip_relative(mem, insn_ram); 1064 test_shld_shrd(mem); 1065 //test_lgdt_lidt(mem); 1066 test_sreg(mem); 1067 test_lldt(mem); 1068 test_ltr(mem); 1069 test_cmov(mem); 1070 1071 test_mmx_movq_mf(mem, insn_page, alt_insn_page, insn_ram); 1072 test_movabs(mem, insn_page, alt_insn_page, insn_ram); 1073 test_smsw_reg(mem, insn_page, alt_insn_page, insn_ram); 1074 test_nop(mem, insn_page, alt_insn_page, insn_ram); 1075 test_crosspage_mmio(mem); 1076 1077 test_string_io_mmio(mem); 1078 1079 return report_summary(); 1080 } 1081