1 2 #include "libcflat.h" 3 #include "desc.h" 4 #include "processor.h" 5 #include "asm/page.h" 6 #include "x86/vm.h" 7 8 #define smp_id() 0 9 10 #define true 1 11 #define false 0 12 13 static _Bool verbose = false; 14 15 typedef unsigned long pt_element_t; 16 static int cpuid_7_ebx; 17 static int cpuid_7_ecx; 18 static int invalid_mask; 19 static int page_table_levels; 20 21 #define PT_BASE_ADDR_MASK ((pt_element_t)((((pt_element_t)1 << 40) - 1) & PAGE_MASK)) 22 #define PT_PSE_BASE_ADDR_MASK (PT_BASE_ADDR_MASK & ~(1ull << 21)) 23 24 #define CR0_WP_MASK (1UL << 16) 25 #define CR4_SMEP_MASK (1UL << 20) 26 27 #define PFERR_PRESENT_MASK (1U << 0) 28 #define PFERR_WRITE_MASK (1U << 1) 29 #define PFERR_USER_MASK (1U << 2) 30 #define PFERR_RESERVED_MASK (1U << 3) 31 #define PFERR_FETCH_MASK (1U << 4) 32 #define PFERR_PK_MASK (1U << 5) 33 34 #define MSR_EFER 0xc0000080 35 #define EFER_NX_MASK (1ull << 11) 36 37 #define PT_INDEX(address, level) \ 38 ((address) >> (12 + ((level)-1) * 9)) & 511 39 40 /* 41 * page table access check tests 42 */ 43 44 enum { 45 AC_PTE_PRESENT_BIT, 46 AC_PTE_WRITABLE_BIT, 47 AC_PTE_USER_BIT, 48 AC_PTE_ACCESSED_BIT, 49 AC_PTE_DIRTY_BIT, 50 AC_PTE_NX_BIT, 51 AC_PTE_BIT51_BIT, 52 53 AC_PDE_PRESENT_BIT, 54 AC_PDE_WRITABLE_BIT, 55 AC_PDE_USER_BIT, 56 AC_PDE_ACCESSED_BIT, 57 AC_PDE_DIRTY_BIT, 58 AC_PDE_PSE_BIT, 59 AC_PDE_NX_BIT, 60 AC_PDE_BIT51_BIT, 61 AC_PDE_BIT13_BIT, 62 63 AC_PKU_AD_BIT, 64 AC_PKU_WD_BIT, 65 AC_PKU_PKEY_BIT, 66 67 AC_ACCESS_USER_BIT, 68 AC_ACCESS_WRITE_BIT, 69 AC_ACCESS_FETCH_BIT, 70 AC_ACCESS_TWICE_BIT, 71 72 AC_CPU_EFER_NX_BIT, 73 AC_CPU_CR0_WP_BIT, 74 AC_CPU_CR4_SMEP_BIT, 75 AC_CPU_CR4_PKE_BIT, 76 77 NR_AC_FLAGS 78 }; 79 80 #define AC_PTE_PRESENT_MASK (1 << AC_PTE_PRESENT_BIT) 81 #define AC_PTE_WRITABLE_MASK (1 << AC_PTE_WRITABLE_BIT) 82 #define AC_PTE_USER_MASK (1 << AC_PTE_USER_BIT) 83 #define AC_PTE_ACCESSED_MASK (1 << AC_PTE_ACCESSED_BIT) 84 #define AC_PTE_DIRTY_MASK (1 << AC_PTE_DIRTY_BIT) 85 #define AC_PTE_NX_MASK (1 << AC_PTE_NX_BIT) 86 #define AC_PTE_BIT51_MASK (1 << AC_PTE_BIT51_BIT) 87 88 #define AC_PDE_PRESENT_MASK (1 << AC_PDE_PRESENT_BIT) 89 #define AC_PDE_WRITABLE_MASK (1 << AC_PDE_WRITABLE_BIT) 90 #define AC_PDE_USER_MASK (1 << AC_PDE_USER_BIT) 91 #define AC_PDE_ACCESSED_MASK (1 << AC_PDE_ACCESSED_BIT) 92 #define AC_PDE_DIRTY_MASK (1 << AC_PDE_DIRTY_BIT) 93 #define AC_PDE_PSE_MASK (1 << AC_PDE_PSE_BIT) 94 #define AC_PDE_NX_MASK (1 << AC_PDE_NX_BIT) 95 #define AC_PDE_BIT51_MASK (1 << AC_PDE_BIT51_BIT) 96 #define AC_PDE_BIT13_MASK (1 << AC_PDE_BIT13_BIT) 97 98 #define AC_PKU_AD_MASK (1 << AC_PKU_AD_BIT) 99 #define AC_PKU_WD_MASK (1 << AC_PKU_WD_BIT) 100 #define AC_PKU_PKEY_MASK (1 << AC_PKU_PKEY_BIT) 101 102 #define AC_ACCESS_USER_MASK (1 << AC_ACCESS_USER_BIT) 103 #define AC_ACCESS_WRITE_MASK (1 << AC_ACCESS_WRITE_BIT) 104 #define AC_ACCESS_FETCH_MASK (1 << AC_ACCESS_FETCH_BIT) 105 #define AC_ACCESS_TWICE_MASK (1 << AC_ACCESS_TWICE_BIT) 106 107 #define AC_CPU_EFER_NX_MASK (1 << AC_CPU_EFER_NX_BIT) 108 #define AC_CPU_CR0_WP_MASK (1 << AC_CPU_CR0_WP_BIT) 109 #define AC_CPU_CR4_SMEP_MASK (1 << AC_CPU_CR4_SMEP_BIT) 110 #define AC_CPU_CR4_PKE_MASK (1 << AC_CPU_CR4_PKE_BIT) 111 112 const char *ac_names[] = { 113 [AC_PTE_PRESENT_BIT] = "pte.p", 114 [AC_PTE_ACCESSED_BIT] = "pte.a", 115 [AC_PTE_WRITABLE_BIT] = "pte.rw", 116 [AC_PTE_USER_BIT] = "pte.user", 117 [AC_PTE_DIRTY_BIT] = "pte.d", 118 [AC_PTE_NX_BIT] = "pte.nx", 119 [AC_PTE_BIT51_BIT] = "pte.51", 120 [AC_PDE_PRESENT_BIT] = "pde.p", 121 [AC_PDE_ACCESSED_BIT] = "pde.a", 122 [AC_PDE_WRITABLE_BIT] = "pde.rw", 123 [AC_PDE_USER_BIT] = "pde.user", 124 [AC_PDE_DIRTY_BIT] = "pde.d", 125 [AC_PDE_PSE_BIT] = "pde.pse", 126 [AC_PDE_NX_BIT] = "pde.nx", 127 [AC_PDE_BIT51_BIT] = "pde.51", 128 [AC_PDE_BIT13_BIT] = "pde.13", 129 [AC_PKU_AD_BIT] = "pkru.ad", 130 [AC_PKU_WD_BIT] = "pkru.wd", 131 [AC_PKU_PKEY_BIT] = "pkey=1", 132 [AC_ACCESS_WRITE_BIT] = "write", 133 [AC_ACCESS_USER_BIT] = "user", 134 [AC_ACCESS_FETCH_BIT] = "fetch", 135 [AC_ACCESS_TWICE_BIT] = "twice", 136 [AC_CPU_EFER_NX_BIT] = "efer.nx", 137 [AC_CPU_CR0_WP_BIT] = "cr0.wp", 138 [AC_CPU_CR4_SMEP_BIT] = "cr4.smep", 139 [AC_CPU_CR4_PKE_BIT] = "cr4.pke", 140 }; 141 142 static inline void *va(pt_element_t phys) 143 { 144 return (void *)phys; 145 } 146 147 typedef struct { 148 pt_element_t pt_pool; 149 unsigned pt_pool_size; 150 unsigned pt_pool_current; 151 } ac_pool_t; 152 153 typedef struct { 154 unsigned flags; 155 void *virt; 156 pt_element_t phys; 157 pt_element_t *ptep; 158 pt_element_t expected_pte; 159 pt_element_t *pdep; 160 pt_element_t expected_pde; 161 pt_element_t ignore_pde; 162 int expected_fault; 163 unsigned expected_error; 164 } ac_test_t; 165 166 typedef struct { 167 unsigned short limit; 168 unsigned long linear_addr; 169 } __attribute__((packed)) descriptor_table_t; 170 171 172 static void ac_test_show(ac_test_t *at); 173 174 static void set_cr0_wp(int wp) 175 { 176 unsigned long cr0 = read_cr0(); 177 unsigned long old_cr0 = cr0; 178 179 cr0 &= ~CR0_WP_MASK; 180 if (wp) 181 cr0 |= CR0_WP_MASK; 182 if (old_cr0 != cr0) 183 write_cr0(cr0); 184 } 185 186 static unsigned set_cr4_smep(int smep) 187 { 188 unsigned long cr4 = read_cr4(); 189 unsigned long old_cr4 = cr4; 190 extern u64 ptl2[]; 191 unsigned r; 192 193 cr4 &= ~CR4_SMEP_MASK; 194 if (smep) 195 cr4 |= CR4_SMEP_MASK; 196 if (old_cr4 == cr4) 197 return 0; 198 199 if (smep) 200 ptl2[2] &= ~PT_USER_MASK; 201 r = write_cr4_checking(cr4); 202 if (r || !smep) 203 ptl2[2] |= PT_USER_MASK; 204 return r; 205 } 206 207 static void set_cr4_pke(int pke) 208 { 209 unsigned long cr4 = read_cr4(); 210 unsigned long old_cr4 = cr4; 211 212 cr4 &= ~X86_CR4_PKE; 213 if (pke) 214 cr4 |= X86_CR4_PKE; 215 if (old_cr4 == cr4) 216 return; 217 218 /* Check that protection keys do not affect accesses when CR4.PKE=0. */ 219 if ((read_cr4() & X86_CR4_PKE) && !pke) { 220 write_pkru(0xfffffffc); 221 } 222 write_cr4(cr4); 223 } 224 225 static void set_efer_nx(int nx) 226 { 227 unsigned long long efer = rdmsr(MSR_EFER); 228 unsigned long long old_efer = efer; 229 230 efer &= ~EFER_NX_MASK; 231 if (nx) 232 efer |= EFER_NX_MASK; 233 if (old_efer != efer) 234 wrmsr(MSR_EFER, efer); 235 } 236 237 static void ac_env_int(ac_pool_t *pool) 238 { 239 extern char page_fault, kernel_entry; 240 set_idt_entry(14, &page_fault, 0); 241 set_idt_entry(0x20, &kernel_entry, 3); 242 243 pool->pt_pool = 33 * 1024 * 1024; 244 pool->pt_pool_size = 120 * 1024 * 1024 - pool->pt_pool; 245 pool->pt_pool_current = 0; 246 } 247 248 static void ac_test_init(ac_test_t *at, void *virt) 249 { 250 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK); 251 set_cr0_wp(1); 252 at->flags = 0; 253 at->virt = virt; 254 at->phys = 32 * 1024 * 1024; 255 } 256 257 static int ac_test_bump_one(ac_test_t *at) 258 { 259 at->flags = ((at->flags | invalid_mask) + 1) & ~invalid_mask; 260 return at->flags < (1 << NR_AC_FLAGS); 261 } 262 263 #define F(x) ((flags & x##_MASK) != 0) 264 265 static _Bool ac_test_legal(ac_test_t *at) 266 { 267 int flags = at->flags; 268 269 if (F(AC_ACCESS_FETCH) && F(AC_ACCESS_WRITE)) 270 return false; 271 272 /* 273 * Since we convert current page to kernel page when cr4.smep=1, 274 * we can't switch to user mode. 275 */ 276 if (F(AC_ACCESS_USER) && F(AC_CPU_CR4_SMEP)) 277 return false; 278 279 /* 280 * Only test protection key faults if CR4.PKE=1. 281 */ 282 if (!F(AC_CPU_CR4_PKE) && 283 (F(AC_PKU_AD) || F(AC_PKU_WD))) { 284 return false; 285 } 286 287 /* 288 * pde.bit13 checks handling of reserved bits in largepage PDEs. It is 289 * meaningless if there is a PTE. 290 */ 291 if (!F(AC_PDE_PSE) && F(AC_PDE_BIT13)) 292 return false; 293 294 return true; 295 } 296 297 static int ac_test_bump(ac_test_t *at) 298 { 299 int ret; 300 301 ret = ac_test_bump_one(at); 302 while (ret && !ac_test_legal(at)) 303 ret = ac_test_bump_one(at); 304 return ret; 305 } 306 307 static pt_element_t ac_test_alloc_pt(ac_pool_t *pool) 308 { 309 pt_element_t ret = pool->pt_pool + pool->pt_pool_current; 310 pool->pt_pool_current += PAGE_SIZE; 311 return ret; 312 } 313 314 static _Bool ac_test_enough_room(ac_pool_t *pool) 315 { 316 return pool->pt_pool_current + 5 * PAGE_SIZE <= pool->pt_pool_size; 317 } 318 319 static void ac_test_reset_pt_pool(ac_pool_t *pool) 320 { 321 pool->pt_pool_current = 0; 322 } 323 324 static pt_element_t ac_test_permissions(ac_test_t *at, unsigned flags, 325 bool writable, bool user, 326 bool executable) 327 { 328 bool kwritable = !F(AC_CPU_CR0_WP) && !F(AC_ACCESS_USER); 329 pt_element_t expected = 0; 330 331 if (F(AC_ACCESS_USER) && !user) 332 at->expected_fault = 1; 333 334 if (F(AC_ACCESS_WRITE) && !writable && !kwritable) 335 at->expected_fault = 1; 336 337 if (F(AC_ACCESS_FETCH) && !executable) 338 at->expected_fault = 1; 339 340 if (F(AC_ACCESS_FETCH) && user && F(AC_CPU_CR4_SMEP)) 341 at->expected_fault = 1; 342 343 if (user && !F(AC_ACCESS_FETCH) && F(AC_PKU_PKEY) && F(AC_CPU_CR4_PKE)) { 344 if (F(AC_PKU_AD)) { 345 at->expected_fault = 1; 346 at->expected_error |= PFERR_PK_MASK; 347 } else if (F(AC_ACCESS_WRITE) && F(AC_PKU_WD) && !kwritable) { 348 at->expected_fault = 1; 349 at->expected_error |= PFERR_PK_MASK; 350 } 351 } 352 353 if (!at->expected_fault) { 354 expected |= PT_ACCESSED_MASK; 355 if (F(AC_ACCESS_WRITE)) 356 expected |= PT_DIRTY_MASK; 357 } 358 359 return expected; 360 } 361 362 static void ac_emulate_access(ac_test_t *at, unsigned flags) 363 { 364 bool pde_valid, pte_valid; 365 bool user, writable, executable; 366 367 if (F(AC_ACCESS_USER)) 368 at->expected_error |= PFERR_USER_MASK; 369 370 if (F(AC_ACCESS_WRITE)) 371 at->expected_error |= PFERR_WRITE_MASK; 372 373 if (F(AC_ACCESS_FETCH)) 374 at->expected_error |= PFERR_FETCH_MASK; 375 376 if (!F(AC_PDE_ACCESSED)) 377 at->ignore_pde = PT_ACCESSED_MASK; 378 379 pde_valid = F(AC_PDE_PRESENT) 380 && !F(AC_PDE_BIT51) && !F(AC_PDE_BIT13) 381 && !(F(AC_PDE_NX) && !F(AC_CPU_EFER_NX)); 382 383 if (!pde_valid) { 384 at->expected_fault = 1; 385 if (F(AC_PDE_PRESENT)) { 386 at->expected_error |= PFERR_RESERVED_MASK; 387 } else { 388 at->expected_error &= ~PFERR_PRESENT_MASK; 389 } 390 goto fault; 391 } 392 393 writable = F(AC_PDE_WRITABLE); 394 user = F(AC_PDE_USER); 395 executable = !F(AC_PDE_NX); 396 397 if (F(AC_PDE_PSE)) { 398 at->expected_pde |= ac_test_permissions(at, flags, writable, user, 399 executable); 400 goto no_pte; 401 } 402 403 at->expected_pde |= PT_ACCESSED_MASK; 404 405 pte_valid = F(AC_PTE_PRESENT) 406 && !F(AC_PTE_BIT51) 407 && !(F(AC_PTE_NX) && !F(AC_CPU_EFER_NX)); 408 409 if (!pte_valid) { 410 at->expected_fault = 1; 411 if (F(AC_PTE_PRESENT)) { 412 at->expected_error |= PFERR_RESERVED_MASK; 413 } else { 414 at->expected_error &= ~PFERR_PRESENT_MASK; 415 } 416 goto fault; 417 } 418 419 writable &= F(AC_PTE_WRITABLE); 420 user &= F(AC_PTE_USER); 421 executable &= !F(AC_PTE_NX); 422 423 at->expected_pte |= ac_test_permissions(at, flags, writable, user, 424 executable); 425 426 no_pte: 427 fault: 428 if (!at->expected_fault) 429 at->ignore_pde = 0; 430 if (!F(AC_CPU_EFER_NX) && !F(AC_CPU_CR4_SMEP)) 431 at->expected_error &= ~PFERR_FETCH_MASK; 432 } 433 434 static void ac_set_expected_status(ac_test_t *at) 435 { 436 invlpg(at->virt); 437 438 if (at->ptep) 439 at->expected_pte = *at->ptep; 440 at->expected_pde = *at->pdep; 441 at->ignore_pde = 0; 442 at->expected_fault = 0; 443 at->expected_error = PFERR_PRESENT_MASK; 444 445 if (at->flags & AC_ACCESS_TWICE_MASK) { 446 ac_emulate_access(at, at->flags & ~AC_ACCESS_WRITE_MASK 447 & ~AC_ACCESS_FETCH_MASK & ~AC_ACCESS_USER_MASK); 448 at->expected_fault = 0; 449 at->expected_error = PFERR_PRESENT_MASK; 450 at->ignore_pde = 0; 451 } 452 453 ac_emulate_access(at, at->flags); 454 } 455 456 static void __ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool, 457 u64 pd_page, u64 pt_page) 458 459 { 460 unsigned long root = read_cr3(); 461 int flags = at->flags; 462 bool skip = true; 463 464 if (!ac_test_enough_room(pool)) 465 ac_test_reset_pt_pool(pool); 466 467 at->ptep = 0; 468 for (int i = page_table_levels; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) { 469 pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK); 470 unsigned index = PT_INDEX((unsigned long)at->virt, i); 471 pt_element_t pte = 0; 472 473 /* 474 * Reuse existing page tables along the path to the test code and data 475 * (which is in the bottom 2MB). 476 */ 477 if (skip && i >= 2 && index == 0) { 478 goto next; 479 } 480 skip = false; 481 482 switch (i) { 483 case 5: 484 case 4: 485 case 3: 486 pte = pd_page ? pd_page : ac_test_alloc_pt(pool); 487 pte |= PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 488 break; 489 case 2: 490 if (!F(AC_PDE_PSE)) { 491 pte = pt_page ? pt_page : ac_test_alloc_pt(pool); 492 /* The protection key is ignored on non-leaf entries. */ 493 if (F(AC_PKU_PKEY)) 494 pte |= 2ull << 59; 495 } else { 496 pte = at->phys & PT_PSE_BASE_ADDR_MASK; 497 pte |= PT_PAGE_SIZE_MASK; 498 if (F(AC_PKU_PKEY)) 499 pte |= 1ull << 59; 500 } 501 if (F(AC_PDE_PRESENT)) 502 pte |= PT_PRESENT_MASK; 503 if (F(AC_PDE_WRITABLE)) 504 pte |= PT_WRITABLE_MASK; 505 if (F(AC_PDE_USER)) 506 pte |= PT_USER_MASK; 507 if (F(AC_PDE_ACCESSED)) 508 pte |= PT_ACCESSED_MASK; 509 if (F(AC_PDE_DIRTY)) 510 pte |= PT_DIRTY_MASK; 511 if (F(AC_PDE_NX)) 512 pte |= PT64_NX_MASK; 513 if (F(AC_PDE_BIT51)) 514 pte |= 1ull << 51; 515 if (F(AC_PDE_BIT13)) 516 pte |= 1ull << 13; 517 at->pdep = &vroot[index]; 518 break; 519 case 1: 520 pte = at->phys & PT_BASE_ADDR_MASK; 521 if (F(AC_PKU_PKEY)) 522 pte |= 1ull << 59; 523 if (F(AC_PTE_PRESENT)) 524 pte |= PT_PRESENT_MASK; 525 if (F(AC_PTE_WRITABLE)) 526 pte |= PT_WRITABLE_MASK; 527 if (F(AC_PTE_USER)) 528 pte |= PT_USER_MASK; 529 if (F(AC_PTE_ACCESSED)) 530 pte |= PT_ACCESSED_MASK; 531 if (F(AC_PTE_DIRTY)) 532 pte |= PT_DIRTY_MASK; 533 if (F(AC_PTE_NX)) 534 pte |= PT64_NX_MASK; 535 if (F(AC_PTE_BIT51)) 536 pte |= 1ull << 51; 537 at->ptep = &vroot[index]; 538 break; 539 } 540 vroot[index] = pte; 541 next: 542 root = vroot[index]; 543 } 544 ac_set_expected_status(at); 545 } 546 547 static void ac_test_setup_pte(ac_test_t *at, ac_pool_t *pool) 548 { 549 __ac_setup_specific_pages(at, pool, 0, 0); 550 } 551 552 static void ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool, 553 u64 pd_page, u64 pt_page) 554 { 555 return __ac_setup_specific_pages(at, pool, pd_page, pt_page); 556 } 557 558 static void dump_mapping(ac_test_t *at) 559 { 560 unsigned long root = read_cr3(); 561 int flags = at->flags; 562 int i; 563 564 printf("Dump mapping: address: %p\n", at->virt); 565 for (i = page_table_levels ; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) { 566 pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK); 567 unsigned index = PT_INDEX((unsigned long)at->virt, i); 568 pt_element_t pte = vroot[index]; 569 570 printf("------L%d: %lx\n", i, pte); 571 root = vroot[index]; 572 } 573 } 574 575 static void ac_test_check(ac_test_t *at, _Bool *success_ret, _Bool cond, 576 const char *fmt, ...) 577 { 578 va_list ap; 579 char buf[500]; 580 581 if (!*success_ret) { 582 return; 583 } 584 585 if (!cond) { 586 return; 587 } 588 589 *success_ret = false; 590 591 if (!verbose) { 592 puts("\n"); 593 ac_test_show(at); 594 } 595 596 va_start(ap, fmt); 597 vsnprintf(buf, sizeof(buf), fmt, ap); 598 va_end(ap); 599 printf("FAIL: %s\n", buf); 600 dump_mapping(at); 601 } 602 603 static int pt_match(pt_element_t pte1, pt_element_t pte2, pt_element_t ignore) 604 { 605 pte1 &= ~ignore; 606 pte2 &= ~ignore; 607 return pte1 == pte2; 608 } 609 610 static int ac_test_do_access(ac_test_t *at) 611 { 612 static unsigned unique = 42; 613 int fault = 0; 614 unsigned e; 615 static unsigned char user_stack[4096]; 616 unsigned long rsp; 617 _Bool success = true; 618 int flags = at->flags; 619 620 ++unique; 621 if (!(unique & 65535)) { 622 puts("."); 623 } 624 625 *((unsigned char *)at->phys) = 0xc3; /* ret */ 626 627 unsigned r = unique; 628 set_cr0_wp(F(AC_CPU_CR0_WP)); 629 set_efer_nx(F(AC_CPU_EFER_NX)); 630 set_cr4_pke(F(AC_CPU_CR4_PKE)); 631 if (F(AC_CPU_CR4_PKE)) { 632 /* WD2=AD2=1, WD1=F(AC_PKU_WD), AD1=F(AC_PKU_AD) */ 633 write_pkru(0x30 | (F(AC_PKU_WD) ? 8 : 0) | 634 (F(AC_PKU_AD) ? 4 : 0)); 635 } 636 637 set_cr4_smep(F(AC_CPU_CR4_SMEP)); 638 639 if (F(AC_ACCESS_TWICE)) { 640 asm volatile ( 641 "mov $fixed2, %%rsi \n\t" 642 "mov (%[addr]), %[reg] \n\t" 643 "fixed2:" 644 : [reg]"=r"(r), [fault]"=a"(fault), "=b"(e) 645 : [addr]"r"(at->virt) 646 : "rsi" 647 ); 648 fault = 0; 649 } 650 651 asm volatile ("mov $fixed1, %%rsi \n\t" 652 "mov %%rsp, %%rdx \n\t" 653 "cmp $0, %[user] \n\t" 654 "jz do_access \n\t" 655 "push %%rax; mov %[user_ds], %%ax; mov %%ax, %%ds; pop %%rax \n\t" 656 "pushq %[user_ds] \n\t" 657 "pushq %[user_stack_top] \n\t" 658 "pushfq \n\t" 659 "pushq %[user_cs] \n\t" 660 "pushq $do_access \n\t" 661 "iretq \n" 662 "do_access: \n\t" 663 "cmp $0, %[fetch] \n\t" 664 "jnz 2f \n\t" 665 "cmp $0, %[write] \n\t" 666 "jnz 1f \n\t" 667 "mov (%[addr]), %[reg] \n\t" 668 "jmp done \n\t" 669 "1: mov %[reg], (%[addr]) \n\t" 670 "jmp done \n\t" 671 "2: call *%[addr] \n\t" 672 "done: \n" 673 "fixed1: \n" 674 "int %[kernel_entry_vector] \n\t" 675 "back_to_kernel:" 676 : [reg]"+r"(r), "+a"(fault), "=b"(e), "=&d"(rsp) 677 : [addr]"r"(at->virt), 678 [write]"r"(F(AC_ACCESS_WRITE)), 679 [user]"r"(F(AC_ACCESS_USER)), 680 [fetch]"r"(F(AC_ACCESS_FETCH)), 681 [user_ds]"i"(USER_DS), 682 [user_cs]"i"(USER_CS), 683 [user_stack_top]"r"(user_stack + sizeof user_stack), 684 [kernel_entry_vector]"i"(0x20) 685 : "rsi"); 686 687 asm volatile (".section .text.pf \n\t" 688 "page_fault: \n\t" 689 "pop %rbx \n\t" 690 "mov %rsi, (%rsp) \n\t" 691 "movl $1, %eax \n\t" 692 "iretq \n\t" 693 ".section .text"); 694 695 asm volatile (".section .text.entry \n\t" 696 "kernel_entry: \n\t" 697 "mov %rdx, %rsp \n\t" 698 "jmp back_to_kernel \n\t" 699 ".section .text"); 700 701 ac_test_check(at, &success, fault && !at->expected_fault, 702 "unexpected fault"); 703 ac_test_check(at, &success, !fault && at->expected_fault, 704 "unexpected access"); 705 ac_test_check(at, &success, fault && e != at->expected_error, 706 "error code %x expected %x", e, at->expected_error); 707 if (at->ptep) 708 ac_test_check(at, &success, *at->ptep != at->expected_pte, 709 "pte %x expected %x", *at->ptep, at->expected_pte); 710 ac_test_check(at, &success, 711 !pt_match(*at->pdep, at->expected_pde, at->ignore_pde), 712 "pde %x expected %x", *at->pdep, at->expected_pde); 713 714 if (success && verbose) { 715 if (at->expected_fault) { 716 printf("PASS (%x)\n", at->expected_error); 717 } else { 718 printf("PASS\n"); 719 } 720 } 721 return success; 722 } 723 724 static void ac_test_show(ac_test_t *at) 725 { 726 char line[5000]; 727 728 *line = 0; 729 strcat(line, "test"); 730 for (int i = 0; i < NR_AC_FLAGS; ++i) 731 if (at->flags & (1 << i)) { 732 strcat(line, " "); 733 strcat(line, ac_names[i]); 734 } 735 strcat(line, ": "); 736 printf("%s", line); 737 } 738 739 /* 740 * This test case is used to triger the bug which is fixed by 741 * commit e09e90a5 in the kvm tree 742 */ 743 static int corrupt_hugepage_triger(ac_pool_t *pool) 744 { 745 ac_test_t at1, at2; 746 747 ac_test_init(&at1, (void *)(0x123400000000)); 748 ac_test_init(&at2, (void *)(0x666600000000)); 749 750 at2.flags = AC_CPU_CR0_WP_MASK | AC_PDE_PSE_MASK | AC_PDE_PRESENT_MASK; 751 ac_test_setup_pte(&at2, pool); 752 if (!ac_test_do_access(&at2)) 753 goto err; 754 755 at1.flags = at2.flags | AC_PDE_WRITABLE_MASK; 756 ac_test_setup_pte(&at1, pool); 757 if (!ac_test_do_access(&at1)) 758 goto err; 759 760 at1.flags |= AC_ACCESS_WRITE_MASK; 761 ac_set_expected_status(&at1); 762 if (!ac_test_do_access(&at1)) 763 goto err; 764 765 at2.flags |= AC_ACCESS_WRITE_MASK; 766 ac_set_expected_status(&at2); 767 if (!ac_test_do_access(&at2)) 768 goto err; 769 770 return 1; 771 772 err: 773 printf("corrupt_hugepage_triger test fail\n"); 774 return 0; 775 } 776 777 /* 778 * This test case is used to triger the bug which is fixed by 779 * commit 3ddf6c06e13e in the kvm tree 780 */ 781 static int check_pfec_on_prefetch_pte(ac_pool_t *pool) 782 { 783 ac_test_t at1, at2; 784 785 ac_test_init(&at1, (void *)(0x123406001000)); 786 ac_test_init(&at2, (void *)(0x123406003000)); 787 788 at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK; 789 ac_setup_specific_pages(&at1, pool, 30 * 1024 * 1024, 30 * 1024 * 1024); 790 791 at2.flags = at1.flags | AC_PTE_NX_MASK; 792 ac_setup_specific_pages(&at2, pool, 30 * 1024 * 1024, 30 * 1024 * 1024); 793 794 if (!ac_test_do_access(&at1)) { 795 printf("%s: prepare fail\n", __FUNCTION__); 796 goto err; 797 } 798 799 if (!ac_test_do_access(&at2)) { 800 printf("%s: check PFEC on prefetch pte path fail\n", 801 __FUNCTION__); 802 goto err; 803 } 804 805 return 1; 806 807 err: 808 return 0; 809 } 810 811 /* 812 * If the write-fault access is from supervisor and CR0.WP is not set on the 813 * vcpu, kvm will fix it by adjusting pte access - it sets the W bit on pte 814 * and clears U bit. This is the chance that kvm can change pte access from 815 * readonly to writable. 816 * 817 * Unfortunately, the pte access is the access of 'direct' shadow page table, 818 * means direct sp.role.access = pte_access, then we will create a writable 819 * spte entry on the readonly shadow page table. It will cause Dirty bit is 820 * not tracked when two guest ptes point to the same large page. Note, it 821 * does not have other impact except Dirty bit since cr0.wp is encoded into 822 * sp.role. 823 * 824 * Note: to trigger this bug, hugepage should be disabled on host. 825 */ 826 static int check_large_pte_dirty_for_nowp(ac_pool_t *pool) 827 { 828 ac_test_t at1, at2; 829 830 ac_test_init(&at1, (void *)(0x123403000000)); 831 ac_test_init(&at2, (void *)(0x666606000000)); 832 833 at2.flags = AC_PDE_PRESENT_MASK | AC_PDE_PSE_MASK; 834 ac_test_setup_pte(&at2, pool); 835 if (!ac_test_do_access(&at2)) { 836 printf("%s: read on the first mapping fail.\n", __FUNCTION__); 837 goto err; 838 } 839 840 at1.flags = at2.flags | AC_ACCESS_WRITE_MASK; 841 ac_test_setup_pte(&at1, pool); 842 if (!ac_test_do_access(&at1)) { 843 printf("%s: write on the second mapping fail.\n", __FUNCTION__); 844 goto err; 845 } 846 847 at2.flags |= AC_ACCESS_WRITE_MASK; 848 ac_set_expected_status(&at2); 849 if (!ac_test_do_access(&at2)) { 850 printf("%s: write on the first mapping fail.\n", __FUNCTION__); 851 goto err; 852 } 853 854 return 1; 855 856 err: 857 return 0; 858 } 859 860 static int check_smep_andnot_wp(ac_pool_t *pool) 861 { 862 ac_test_t at1; 863 int err_prepare_andnot_wp, err_smep_andnot_wp; 864 865 if (!(cpuid_7_ebx & (1 << 7))) { 866 return 1; 867 } 868 869 ac_test_init(&at1, (void *)(0x123406001000)); 870 871 at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK | 872 AC_PDE_USER_MASK | AC_PTE_USER_MASK | 873 AC_PDE_ACCESSED_MASK | AC_PTE_ACCESSED_MASK | 874 AC_CPU_CR4_SMEP_MASK | 875 AC_CPU_CR0_WP_MASK | 876 AC_ACCESS_WRITE_MASK; 877 ac_test_setup_pte(&at1, pool); 878 879 /* 880 * Here we write the ro user page when 881 * cr0.wp=0, then we execute it and SMEP 882 * fault should happen. 883 */ 884 err_prepare_andnot_wp = ac_test_do_access(&at1); 885 if (!err_prepare_andnot_wp) { 886 printf("%s: SMEP prepare fail\n", __FUNCTION__); 887 goto clean_up; 888 } 889 890 at1.flags &= ~AC_ACCESS_WRITE_MASK; 891 at1.flags |= AC_ACCESS_FETCH_MASK; 892 ac_set_expected_status(&at1); 893 err_smep_andnot_wp = ac_test_do_access(&at1); 894 895 clean_up: 896 set_cr4_smep(0); 897 898 if (!err_prepare_andnot_wp) 899 goto err; 900 if (!err_smep_andnot_wp) { 901 printf("%s: check SMEP without wp fail\n", __FUNCTION__); 902 goto err; 903 } 904 return 1; 905 906 err: 907 return 0; 908 } 909 910 static int ac_test_exec(ac_test_t *at, ac_pool_t *pool) 911 { 912 int r; 913 914 if (verbose) { 915 ac_test_show(at); 916 } 917 ac_test_setup_pte(at, pool); 918 r = ac_test_do_access(at); 919 return r; 920 } 921 922 typedef int (*ac_test_fn)(ac_pool_t *pool); 923 const ac_test_fn ac_test_cases[] = 924 { 925 corrupt_hugepage_triger, 926 check_pfec_on_prefetch_pte, 927 check_large_pte_dirty_for_nowp, 928 check_smep_andnot_wp 929 }; 930 931 static int ac_test_run(void) 932 { 933 ac_test_t at; 934 ac_pool_t pool; 935 int i, tests, successes; 936 937 printf("run\n"); 938 tests = successes = 0; 939 940 if (cpuid_7_ecx & (1 << 3)) { 941 set_cr4_pke(1); 942 set_cr4_pke(0); 943 /* Now PKRU = 0xFFFFFFFF. */ 944 } else { 945 unsigned long cr4 = read_cr4(); 946 tests++; 947 if (write_cr4_checking(cr4 | X86_CR4_PKE) == GP_VECTOR) { 948 successes++; 949 invalid_mask |= AC_PKU_AD_MASK; 950 invalid_mask |= AC_PKU_WD_MASK; 951 invalid_mask |= AC_PKU_PKEY_MASK; 952 invalid_mask |= AC_CPU_CR4_PKE_MASK; 953 printf("CR4.PKE not available, disabling PKE tests\n"); 954 } else { 955 printf("Set PKE in CR4 - expect #GP: FAIL!\n"); 956 set_cr4_pke(0); 957 } 958 } 959 960 if (!(cpuid_7_ebx & (1 << 7))) { 961 tests++; 962 if (set_cr4_smep(1) == GP_VECTOR) { 963 successes++; 964 invalid_mask |= AC_CPU_CR4_SMEP_MASK; 965 printf("CR4.SMEP not available, disabling SMEP tests\n"); 966 } else { 967 printf("Set SMEP in CR4 - expect #GP: FAIL!\n"); 968 set_cr4_smep(0); 969 } 970 } 971 972 ac_env_int(&pool); 973 ac_test_init(&at, (void *)(0x123400000000 + 16 * smp_id())); 974 do { 975 ++tests; 976 successes += ac_test_exec(&at, &pool); 977 } while (ac_test_bump(&at)); 978 979 for (i = 0; i < ARRAY_SIZE(ac_test_cases); i++) { 980 ++tests; 981 successes += ac_test_cases[i](&pool); 982 } 983 984 printf("\n%d tests, %d failures\n", tests, tests - successes); 985 986 return successes == tests; 987 } 988 989 int main(void) 990 { 991 int r; 992 993 setup_idt(); 994 995 cpuid_7_ebx = cpuid(7).b; 996 cpuid_7_ecx = cpuid(7).c; 997 998 printf("starting test\n\n"); 999 page_table_levels = 4; 1000 r = ac_test_run(); 1001 1002 if (cpuid_7_ecx & (1 << 16)) { 1003 page_table_levels = 5; 1004 setup_5level_page_table(); 1005 printf("starting 5-level paging test.\n\n"); 1006 r = ac_test_run(); 1007 } 1008 1009 return r ? 0 : 1; 1010 } 1011