1 2 #include "libcflat.h" 3 #include "desc.h" 4 #include "processor.h" 5 #include "asm/page.h" 6 #include "x86/vm.h" 7 8 #define smp_id() 0 9 10 #define true 1 11 #define false 0 12 13 static _Bool verbose = false; 14 15 typedef unsigned long pt_element_t; 16 static int invalid_mask; 17 static int page_table_levels; 18 19 #define PT_BASE_ADDR_MASK ((pt_element_t)((((pt_element_t)1 << 36) - 1) & PAGE_MASK)) 20 #define PT_PSE_BASE_ADDR_MASK (PT_BASE_ADDR_MASK & ~(1ull << 21)) 21 22 #define CR0_WP_MASK (1UL << 16) 23 #define CR4_SMEP_MASK (1UL << 20) 24 25 #define PFERR_PRESENT_MASK (1U << 0) 26 #define PFERR_WRITE_MASK (1U << 1) 27 #define PFERR_USER_MASK (1U << 2) 28 #define PFERR_RESERVED_MASK (1U << 3) 29 #define PFERR_FETCH_MASK (1U << 4) 30 #define PFERR_PK_MASK (1U << 5) 31 32 #define MSR_EFER 0xc0000080 33 #define EFER_NX_MASK (1ull << 11) 34 35 #define PT_INDEX(address, level) \ 36 ((address) >> (12 + ((level)-1) * 9)) & 511 37 38 /* 39 * page table access check tests 40 */ 41 42 enum { 43 AC_PTE_PRESENT_BIT, 44 AC_PTE_WRITABLE_BIT, 45 AC_PTE_USER_BIT, 46 AC_PTE_ACCESSED_BIT, 47 AC_PTE_DIRTY_BIT, 48 AC_PTE_NX_BIT, 49 AC_PTE_BIT51_BIT, 50 AC_PTE_BIT36_BIT, 51 52 AC_PDE_PRESENT_BIT, 53 AC_PDE_WRITABLE_BIT, 54 AC_PDE_USER_BIT, 55 AC_PDE_ACCESSED_BIT, 56 AC_PDE_DIRTY_BIT, 57 AC_PDE_PSE_BIT, 58 AC_PDE_NX_BIT, 59 AC_PDE_BIT51_BIT, 60 AC_PDE_BIT36_BIT, 61 AC_PDE_BIT13_BIT, 62 63 /* 64 * special test case to DISABLE writable bit on page directory 65 * pointer table entry. 66 */ 67 AC_PDPTE_NO_WRITABLE_BIT, 68 69 AC_PKU_AD_BIT, 70 AC_PKU_WD_BIT, 71 AC_PKU_PKEY_BIT, 72 73 AC_ACCESS_USER_BIT, 74 AC_ACCESS_WRITE_BIT, 75 AC_ACCESS_FETCH_BIT, 76 AC_ACCESS_TWICE_BIT, 77 78 AC_CPU_EFER_NX_BIT, 79 AC_CPU_CR0_WP_BIT, 80 AC_CPU_CR4_SMEP_BIT, 81 AC_CPU_CR4_PKE_BIT, 82 83 NR_AC_FLAGS 84 }; 85 86 #define AC_PTE_PRESENT_MASK (1 << AC_PTE_PRESENT_BIT) 87 #define AC_PTE_WRITABLE_MASK (1 << AC_PTE_WRITABLE_BIT) 88 #define AC_PTE_USER_MASK (1 << AC_PTE_USER_BIT) 89 #define AC_PTE_ACCESSED_MASK (1 << AC_PTE_ACCESSED_BIT) 90 #define AC_PTE_DIRTY_MASK (1 << AC_PTE_DIRTY_BIT) 91 #define AC_PTE_NX_MASK (1 << AC_PTE_NX_BIT) 92 #define AC_PTE_BIT51_MASK (1 << AC_PTE_BIT51_BIT) 93 #define AC_PTE_BIT36_MASK (1 << AC_PTE_BIT36_BIT) 94 95 #define AC_PDE_PRESENT_MASK (1 << AC_PDE_PRESENT_BIT) 96 #define AC_PDE_WRITABLE_MASK (1 << AC_PDE_WRITABLE_BIT) 97 #define AC_PDE_USER_MASK (1 << AC_PDE_USER_BIT) 98 #define AC_PDE_ACCESSED_MASK (1 << AC_PDE_ACCESSED_BIT) 99 #define AC_PDE_DIRTY_MASK (1 << AC_PDE_DIRTY_BIT) 100 #define AC_PDE_PSE_MASK (1 << AC_PDE_PSE_BIT) 101 #define AC_PDE_NX_MASK (1 << AC_PDE_NX_BIT) 102 #define AC_PDE_BIT51_MASK (1 << AC_PDE_BIT51_BIT) 103 #define AC_PDE_BIT36_MASK (1 << AC_PDE_BIT36_BIT) 104 #define AC_PDE_BIT13_MASK (1 << AC_PDE_BIT13_BIT) 105 106 #define AC_PDPTE_NO_WRITABLE_MASK (1 << AC_PDPTE_NO_WRITABLE_BIT) 107 108 #define AC_PKU_AD_MASK (1 << AC_PKU_AD_BIT) 109 #define AC_PKU_WD_MASK (1 << AC_PKU_WD_BIT) 110 #define AC_PKU_PKEY_MASK (1 << AC_PKU_PKEY_BIT) 111 112 #define AC_ACCESS_USER_MASK (1 << AC_ACCESS_USER_BIT) 113 #define AC_ACCESS_WRITE_MASK (1 << AC_ACCESS_WRITE_BIT) 114 #define AC_ACCESS_FETCH_MASK (1 << AC_ACCESS_FETCH_BIT) 115 #define AC_ACCESS_TWICE_MASK (1 << AC_ACCESS_TWICE_BIT) 116 117 #define AC_CPU_EFER_NX_MASK (1 << AC_CPU_EFER_NX_BIT) 118 #define AC_CPU_CR0_WP_MASK (1 << AC_CPU_CR0_WP_BIT) 119 #define AC_CPU_CR4_SMEP_MASK (1 << AC_CPU_CR4_SMEP_BIT) 120 #define AC_CPU_CR4_PKE_MASK (1 << AC_CPU_CR4_PKE_BIT) 121 122 const char *ac_names[] = { 123 [AC_PTE_PRESENT_BIT] = "pte.p", 124 [AC_PTE_ACCESSED_BIT] = "pte.a", 125 [AC_PTE_WRITABLE_BIT] = "pte.rw", 126 [AC_PTE_USER_BIT] = "pte.user", 127 [AC_PTE_DIRTY_BIT] = "pte.d", 128 [AC_PTE_NX_BIT] = "pte.nx", 129 [AC_PTE_BIT51_BIT] = "pte.51", 130 [AC_PTE_BIT36_BIT] = "pte.36", 131 [AC_PDE_PRESENT_BIT] = "pde.p", 132 [AC_PDE_ACCESSED_BIT] = "pde.a", 133 [AC_PDE_WRITABLE_BIT] = "pde.rw", 134 [AC_PDE_USER_BIT] = "pde.user", 135 [AC_PDE_DIRTY_BIT] = "pde.d", 136 [AC_PDE_PSE_BIT] = "pde.pse", 137 [AC_PDE_NX_BIT] = "pde.nx", 138 [AC_PDE_BIT51_BIT] = "pde.51", 139 [AC_PDE_BIT36_BIT] = "pde.36", 140 [AC_PDE_BIT13_BIT] = "pde.13", 141 [AC_PDPTE_NO_WRITABLE_BIT] = "pdpte.ro", 142 [AC_PKU_AD_BIT] = "pkru.ad", 143 [AC_PKU_WD_BIT] = "pkru.wd", 144 [AC_PKU_PKEY_BIT] = "pkey=1", 145 [AC_ACCESS_WRITE_BIT] = "write", 146 [AC_ACCESS_USER_BIT] = "user", 147 [AC_ACCESS_FETCH_BIT] = "fetch", 148 [AC_ACCESS_TWICE_BIT] = "twice", 149 [AC_CPU_EFER_NX_BIT] = "efer.nx", 150 [AC_CPU_CR0_WP_BIT] = "cr0.wp", 151 [AC_CPU_CR4_SMEP_BIT] = "cr4.smep", 152 [AC_CPU_CR4_PKE_BIT] = "cr4.pke", 153 }; 154 155 static inline void *va(pt_element_t phys) 156 { 157 return (void *)phys; 158 } 159 160 typedef struct { 161 pt_element_t pt_pool; 162 unsigned pt_pool_size; 163 unsigned pt_pool_current; 164 } ac_pool_t; 165 166 typedef struct { 167 unsigned flags; 168 void *virt; 169 pt_element_t phys; 170 pt_element_t *ptep; 171 pt_element_t expected_pte; 172 pt_element_t *pdep; 173 pt_element_t expected_pde; 174 pt_element_t ignore_pde; 175 int expected_fault; 176 unsigned expected_error; 177 } ac_test_t; 178 179 typedef struct { 180 unsigned short limit; 181 unsigned long linear_addr; 182 } __attribute__((packed)) descriptor_table_t; 183 184 185 static void ac_test_show(ac_test_t *at); 186 187 static unsigned long shadow_cr0; 188 static unsigned long shadow_cr4; 189 static unsigned long long shadow_efer; 190 191 static void set_cr0_wp(int wp) 192 { 193 unsigned long cr0 = shadow_cr0; 194 195 cr0 &= ~CR0_WP_MASK; 196 if (wp) 197 cr0 |= CR0_WP_MASK; 198 if (cr0 != shadow_cr0) { 199 write_cr0(cr0); 200 shadow_cr0 = cr0; 201 } 202 } 203 204 static unsigned set_cr4_smep(int smep) 205 { 206 unsigned long cr4 = shadow_cr4; 207 extern u64 ptl2[]; 208 unsigned r; 209 210 cr4 &= ~CR4_SMEP_MASK; 211 if (smep) 212 cr4 |= CR4_SMEP_MASK; 213 if (cr4 == shadow_cr4) 214 return 0; 215 216 if (smep) 217 ptl2[2] &= ~PT_USER_MASK; 218 r = write_cr4_checking(cr4); 219 if (r || !smep) 220 ptl2[2] |= PT_USER_MASK; 221 if (!r) 222 shadow_cr4 = cr4; 223 return r; 224 } 225 226 static void set_cr4_pke(int pke) 227 { 228 unsigned long cr4 = shadow_cr4; 229 230 cr4 &= ~X86_CR4_PKE; 231 if (pke) 232 cr4 |= X86_CR4_PKE; 233 if (cr4 == shadow_cr4) 234 return; 235 236 /* Check that protection keys do not affect accesses when CR4.PKE=0. */ 237 if ((shadow_cr4 & X86_CR4_PKE) && !pke) 238 write_pkru(0xfffffffc); 239 write_cr4(cr4); 240 shadow_cr4 = cr4; 241 } 242 243 static void set_efer_nx(int nx) 244 { 245 unsigned long long efer = shadow_efer; 246 247 efer &= ~EFER_NX_MASK; 248 if (nx) 249 efer |= EFER_NX_MASK; 250 if (efer != shadow_efer) { 251 wrmsr(MSR_EFER, efer); 252 shadow_efer = efer; 253 } 254 } 255 256 static void ac_env_int(ac_pool_t *pool) 257 { 258 extern char page_fault, kernel_entry; 259 set_idt_entry(14, &page_fault, 0); 260 set_idt_entry(0x20, &kernel_entry, 3); 261 262 pool->pt_pool = 33 * 1024 * 1024; 263 pool->pt_pool_size = 120 * 1024 * 1024 - pool->pt_pool; 264 pool->pt_pool_current = 0; 265 } 266 267 static void ac_test_init(ac_test_t *at, void *virt) 268 { 269 set_efer_nx(1); 270 set_cr0_wp(1); 271 at->flags = 0; 272 at->virt = virt; 273 at->phys = 32 * 1024 * 1024; 274 } 275 276 static int ac_test_bump_one(ac_test_t *at) 277 { 278 at->flags = ((at->flags | invalid_mask) + 1) & ~invalid_mask; 279 return at->flags < (1 << NR_AC_FLAGS); 280 } 281 282 #define F(x) ((flags & x##_MASK) != 0) 283 284 static _Bool ac_test_legal(ac_test_t *at) 285 { 286 int flags = at->flags; 287 288 if (F(AC_ACCESS_FETCH) && F(AC_ACCESS_WRITE)) 289 return false; 290 291 /* 292 * Since we convert current page to kernel page when cr4.smep=1, 293 * we can't switch to user mode. 294 */ 295 if (F(AC_ACCESS_USER) && F(AC_CPU_CR4_SMEP)) 296 return false; 297 298 /* 299 * Only test protection key faults if CR4.PKE=1. 300 */ 301 if (!F(AC_CPU_CR4_PKE) && 302 (F(AC_PKU_AD) || F(AC_PKU_WD))) { 303 return false; 304 } 305 306 /* 307 * pde.bit13 checks handling of reserved bits in largepage PDEs. It is 308 * meaningless if there is a PTE. 309 */ 310 if (!F(AC_PDE_PSE) && F(AC_PDE_BIT13)) 311 return false; 312 313 /* 314 * Shorten the test by avoiding testing too many reserved bit combinations 315 */ 316 if ((F(AC_PDE_BIT51) + F(AC_PDE_BIT36) + F(AC_PDE_BIT13)) > 1) 317 return false; 318 if ((F(AC_PTE_BIT51) + F(AC_PTE_BIT36)) > 1) 319 return false; 320 321 return true; 322 } 323 324 static int ac_test_bump(ac_test_t *at) 325 { 326 int ret; 327 328 ret = ac_test_bump_one(at); 329 while (ret && !ac_test_legal(at)) 330 ret = ac_test_bump_one(at); 331 return ret; 332 } 333 334 static pt_element_t ac_test_alloc_pt(ac_pool_t *pool) 335 { 336 pt_element_t ret = pool->pt_pool + pool->pt_pool_current; 337 pool->pt_pool_current += PAGE_SIZE; 338 memset(va(ret), 0, PAGE_SIZE); 339 return ret; 340 } 341 342 static _Bool ac_test_enough_room(ac_pool_t *pool) 343 { 344 return pool->pt_pool_current + 5 * PAGE_SIZE <= pool->pt_pool_size; 345 } 346 347 static void ac_test_reset_pt_pool(ac_pool_t *pool) 348 { 349 pool->pt_pool_current = 0; 350 } 351 352 static pt_element_t ac_test_permissions(ac_test_t *at, unsigned flags, 353 bool writable, bool user, 354 bool executable) 355 { 356 bool kwritable = !F(AC_CPU_CR0_WP) && !F(AC_ACCESS_USER); 357 pt_element_t expected = 0; 358 359 if (F(AC_ACCESS_USER) && !user) 360 at->expected_fault = 1; 361 362 if (F(AC_ACCESS_WRITE) && !writable && !kwritable) 363 at->expected_fault = 1; 364 365 if (F(AC_ACCESS_FETCH) && !executable) 366 at->expected_fault = 1; 367 368 if (F(AC_ACCESS_FETCH) && user && F(AC_CPU_CR4_SMEP)) 369 at->expected_fault = 1; 370 371 if (user && !F(AC_ACCESS_FETCH) && F(AC_PKU_PKEY) && F(AC_CPU_CR4_PKE)) { 372 if (F(AC_PKU_AD)) { 373 at->expected_fault = 1; 374 at->expected_error |= PFERR_PK_MASK; 375 } else if (F(AC_ACCESS_WRITE) && F(AC_PKU_WD) && !kwritable) { 376 at->expected_fault = 1; 377 at->expected_error |= PFERR_PK_MASK; 378 } 379 } 380 381 if (!at->expected_fault) { 382 expected |= PT_ACCESSED_MASK; 383 if (F(AC_ACCESS_WRITE)) 384 expected |= PT_DIRTY_MASK; 385 } 386 387 return expected; 388 } 389 390 static void ac_emulate_access(ac_test_t *at, unsigned flags) 391 { 392 bool pde_valid, pte_valid; 393 bool user, writable, executable; 394 395 if (F(AC_ACCESS_USER)) 396 at->expected_error |= PFERR_USER_MASK; 397 398 if (F(AC_ACCESS_WRITE)) 399 at->expected_error |= PFERR_WRITE_MASK; 400 401 if (F(AC_ACCESS_FETCH)) 402 at->expected_error |= PFERR_FETCH_MASK; 403 404 if (!F(AC_PDE_ACCESSED)) 405 at->ignore_pde = PT_ACCESSED_MASK; 406 407 pde_valid = F(AC_PDE_PRESENT) 408 && !F(AC_PDE_BIT51) && !F(AC_PDE_BIT36) && !F(AC_PDE_BIT13) 409 && !(F(AC_PDE_NX) && !F(AC_CPU_EFER_NX)); 410 411 if (!pde_valid) { 412 at->expected_fault = 1; 413 if (F(AC_PDE_PRESENT)) { 414 at->expected_error |= PFERR_RESERVED_MASK; 415 } else { 416 at->expected_error &= ~PFERR_PRESENT_MASK; 417 } 418 goto fault; 419 } 420 421 writable = !F(AC_PDPTE_NO_WRITABLE) && F(AC_PDE_WRITABLE); 422 user = F(AC_PDE_USER); 423 executable = !F(AC_PDE_NX); 424 425 if (F(AC_PDE_PSE)) { 426 at->expected_pde |= ac_test_permissions(at, flags, writable, user, 427 executable); 428 goto no_pte; 429 } 430 431 at->expected_pde |= PT_ACCESSED_MASK; 432 433 pte_valid = F(AC_PTE_PRESENT) 434 && !F(AC_PTE_BIT51) && !F(AC_PTE_BIT36) 435 && !(F(AC_PTE_NX) && !F(AC_CPU_EFER_NX)); 436 437 if (!pte_valid) { 438 at->expected_fault = 1; 439 if (F(AC_PTE_PRESENT)) { 440 at->expected_error |= PFERR_RESERVED_MASK; 441 } else { 442 at->expected_error &= ~PFERR_PRESENT_MASK; 443 } 444 goto fault; 445 } 446 447 writable &= F(AC_PTE_WRITABLE); 448 user &= F(AC_PTE_USER); 449 executable &= !F(AC_PTE_NX); 450 451 at->expected_pte |= ac_test_permissions(at, flags, writable, user, 452 executable); 453 454 no_pte: 455 fault: 456 if (!at->expected_fault) 457 at->ignore_pde = 0; 458 if (!F(AC_CPU_EFER_NX) && !F(AC_CPU_CR4_SMEP)) 459 at->expected_error &= ~PFERR_FETCH_MASK; 460 } 461 462 static void ac_set_expected_status(ac_test_t *at) 463 { 464 invlpg(at->virt); 465 466 if (at->ptep) 467 at->expected_pte = *at->ptep; 468 at->expected_pde = *at->pdep; 469 at->ignore_pde = 0; 470 at->expected_fault = 0; 471 at->expected_error = PFERR_PRESENT_MASK; 472 473 if (at->flags & AC_ACCESS_TWICE_MASK) { 474 ac_emulate_access(at, at->flags & ~AC_ACCESS_WRITE_MASK 475 & ~AC_ACCESS_FETCH_MASK & ~AC_ACCESS_USER_MASK); 476 at->expected_fault = 0; 477 at->expected_error = PFERR_PRESENT_MASK; 478 at->ignore_pde = 0; 479 } 480 481 ac_emulate_access(at, at->flags); 482 } 483 484 static void __ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool, bool reuse, 485 u64 pd_page, u64 pt_page) 486 487 { 488 unsigned long root = read_cr3(); 489 int flags = at->flags; 490 bool skip = true; 491 492 if (!ac_test_enough_room(pool)) 493 ac_test_reset_pt_pool(pool); 494 495 at->ptep = 0; 496 for (int i = page_table_levels; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) { 497 pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK); 498 unsigned index = PT_INDEX((unsigned long)at->virt, i); 499 pt_element_t pte = 0; 500 501 /* 502 * Reuse existing page tables along the path to the test code and data 503 * (which is in the bottom 2MB). 504 */ 505 if (skip && i >= 2 && index == 0) { 506 goto next; 507 } 508 skip = false; 509 if (reuse && vroot[index]) { 510 switch (i) { 511 case 2: 512 at->pdep = &vroot[index]; 513 break; 514 case 1: 515 at->ptep = &vroot[index]; 516 break; 517 } 518 goto next; 519 } 520 521 switch (i) { 522 case 5: 523 case 4: 524 pte = ac_test_alloc_pt(pool); 525 pte |= PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 526 break; 527 case 3: 528 pte = pd_page ? pd_page : ac_test_alloc_pt(pool); 529 pte |= PT_PRESENT_MASK | PT_USER_MASK; 530 if (!F(AC_PDPTE_NO_WRITABLE)) 531 pte |= PT_WRITABLE_MASK; 532 break; 533 case 2: 534 if (!F(AC_PDE_PSE)) { 535 pte = pt_page ? pt_page : ac_test_alloc_pt(pool); 536 /* The protection key is ignored on non-leaf entries. */ 537 if (F(AC_PKU_PKEY)) 538 pte |= 2ull << 59; 539 } else { 540 pte = at->phys & PT_PSE_BASE_ADDR_MASK; 541 pte |= PT_PAGE_SIZE_MASK; 542 if (F(AC_PKU_PKEY)) 543 pte |= 1ull << 59; 544 } 545 if (F(AC_PDE_PRESENT)) 546 pte |= PT_PRESENT_MASK; 547 if (F(AC_PDE_WRITABLE)) 548 pte |= PT_WRITABLE_MASK; 549 if (F(AC_PDE_USER)) 550 pte |= PT_USER_MASK; 551 if (F(AC_PDE_ACCESSED)) 552 pte |= PT_ACCESSED_MASK; 553 if (F(AC_PDE_DIRTY)) 554 pte |= PT_DIRTY_MASK; 555 if (F(AC_PDE_NX)) 556 pte |= PT64_NX_MASK; 557 if (F(AC_PDE_BIT51)) 558 pte |= 1ull << 51; 559 if (F(AC_PDE_BIT36)) 560 pte |= 1ull << 36; 561 if (F(AC_PDE_BIT13)) 562 pte |= 1ull << 13; 563 at->pdep = &vroot[index]; 564 break; 565 case 1: 566 pte = at->phys & PT_BASE_ADDR_MASK; 567 if (F(AC_PKU_PKEY)) 568 pte |= 1ull << 59; 569 if (F(AC_PTE_PRESENT)) 570 pte |= PT_PRESENT_MASK; 571 if (F(AC_PTE_WRITABLE)) 572 pte |= PT_WRITABLE_MASK; 573 if (F(AC_PTE_USER)) 574 pte |= PT_USER_MASK; 575 if (F(AC_PTE_ACCESSED)) 576 pte |= PT_ACCESSED_MASK; 577 if (F(AC_PTE_DIRTY)) 578 pte |= PT_DIRTY_MASK; 579 if (F(AC_PTE_NX)) 580 pte |= PT64_NX_MASK; 581 if (F(AC_PTE_BIT51)) 582 pte |= 1ull << 51; 583 if (F(AC_PTE_BIT36)) 584 pte |= 1ull << 36; 585 at->ptep = &vroot[index]; 586 break; 587 } 588 vroot[index] = pte; 589 next: 590 root = vroot[index]; 591 } 592 ac_set_expected_status(at); 593 } 594 595 static void ac_test_setup_pte(ac_test_t *at, ac_pool_t *pool) 596 { 597 __ac_setup_specific_pages(at, pool, false, 0, 0); 598 } 599 600 static void ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool, 601 u64 pd_page, u64 pt_page) 602 { 603 return __ac_setup_specific_pages(at, pool, false, pd_page, pt_page); 604 } 605 606 static void dump_mapping(ac_test_t *at) 607 { 608 unsigned long root = read_cr3(); 609 int flags = at->flags; 610 int i; 611 612 printf("Dump mapping: address: %p\n", at->virt); 613 for (i = page_table_levels ; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) { 614 pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK); 615 unsigned index = PT_INDEX((unsigned long)at->virt, i); 616 pt_element_t pte = vroot[index]; 617 618 printf("------L%d: %lx\n", i, pte); 619 root = vroot[index]; 620 } 621 } 622 623 static void ac_test_check(ac_test_t *at, _Bool *success_ret, _Bool cond, 624 const char *fmt, ...) 625 { 626 va_list ap; 627 char buf[500]; 628 629 if (!*success_ret) { 630 return; 631 } 632 633 if (!cond) { 634 return; 635 } 636 637 *success_ret = false; 638 639 if (!verbose) { 640 puts("\n"); 641 ac_test_show(at); 642 } 643 644 va_start(ap, fmt); 645 vsnprintf(buf, sizeof(buf), fmt, ap); 646 va_end(ap); 647 printf("FAIL: %s\n", buf); 648 dump_mapping(at); 649 } 650 651 static int pt_match(pt_element_t pte1, pt_element_t pte2, pt_element_t ignore) 652 { 653 pte1 &= ~ignore; 654 pte2 &= ~ignore; 655 return pte1 == pte2; 656 } 657 658 static int ac_test_do_access(ac_test_t *at) 659 { 660 static unsigned unique = 42; 661 int fault = 0; 662 unsigned e; 663 static unsigned char user_stack[4096]; 664 unsigned long rsp; 665 _Bool success = true; 666 int flags = at->flags; 667 668 ++unique; 669 if (!(unique & 65535)) { 670 puts("."); 671 } 672 673 *((unsigned char *)at->phys) = 0xc3; /* ret */ 674 675 unsigned r = unique; 676 set_cr0_wp(F(AC_CPU_CR0_WP)); 677 set_efer_nx(F(AC_CPU_EFER_NX)); 678 set_cr4_pke(F(AC_CPU_CR4_PKE)); 679 if (F(AC_CPU_CR4_PKE)) { 680 /* WD2=AD2=1, WD1=F(AC_PKU_WD), AD1=F(AC_PKU_AD) */ 681 write_pkru(0x30 | (F(AC_PKU_WD) ? 8 : 0) | 682 (F(AC_PKU_AD) ? 4 : 0)); 683 } 684 685 set_cr4_smep(F(AC_CPU_CR4_SMEP)); 686 687 if (F(AC_ACCESS_TWICE)) { 688 asm volatile ( 689 "mov $fixed2, %%rsi \n\t" 690 "mov (%[addr]), %[reg] \n\t" 691 "fixed2:" 692 : [reg]"=r"(r), [fault]"=a"(fault), "=b"(e) 693 : [addr]"r"(at->virt) 694 : "rsi" 695 ); 696 fault = 0; 697 } 698 699 asm volatile ("mov $fixed1, %%rsi \n\t" 700 "mov %%rsp, %%rdx \n\t" 701 "cmp $0, %[user] \n\t" 702 "jz do_access \n\t" 703 "push %%rax; mov %[user_ds], %%ax; mov %%ax, %%ds; pop %%rax \n\t" 704 "pushq %[user_ds] \n\t" 705 "pushq %[user_stack_top] \n\t" 706 "pushfq \n\t" 707 "pushq %[user_cs] \n\t" 708 "pushq $do_access \n\t" 709 "iretq \n" 710 "do_access: \n\t" 711 "cmp $0, %[fetch] \n\t" 712 "jnz 2f \n\t" 713 "cmp $0, %[write] \n\t" 714 "jnz 1f \n\t" 715 "mov (%[addr]), %[reg] \n\t" 716 "jmp done \n\t" 717 "1: mov %[reg], (%[addr]) \n\t" 718 "jmp done \n\t" 719 "2: call *%[addr] \n\t" 720 "done: \n" 721 "fixed1: \n" 722 "int %[kernel_entry_vector] \n\t" 723 "back_to_kernel:" 724 : [reg]"+r"(r), "+a"(fault), "=b"(e), "=&d"(rsp) 725 : [addr]"r"(at->virt), 726 [write]"r"(F(AC_ACCESS_WRITE)), 727 [user]"r"(F(AC_ACCESS_USER)), 728 [fetch]"r"(F(AC_ACCESS_FETCH)), 729 [user_ds]"i"(USER_DS), 730 [user_cs]"i"(USER_CS), 731 [user_stack_top]"r"(user_stack + sizeof user_stack), 732 [kernel_entry_vector]"i"(0x20) 733 : "rsi"); 734 735 asm volatile (".section .text.pf \n\t" 736 "page_fault: \n\t" 737 "pop %rbx \n\t" 738 "mov %rsi, (%rsp) \n\t" 739 "movl $1, %eax \n\t" 740 "iretq \n\t" 741 ".section .text"); 742 743 asm volatile (".section .text.entry \n\t" 744 "kernel_entry: \n\t" 745 "mov %rdx, %rsp \n\t" 746 "jmp back_to_kernel \n\t" 747 ".section .text"); 748 749 ac_test_check(at, &success, fault && !at->expected_fault, 750 "unexpected fault"); 751 ac_test_check(at, &success, !fault && at->expected_fault, 752 "unexpected access"); 753 ac_test_check(at, &success, fault && e != at->expected_error, 754 "error code %x expected %x", e, at->expected_error); 755 if (at->ptep) 756 ac_test_check(at, &success, *at->ptep != at->expected_pte, 757 "pte %x expected %x", *at->ptep, at->expected_pte); 758 ac_test_check(at, &success, 759 !pt_match(*at->pdep, at->expected_pde, at->ignore_pde), 760 "pde %x expected %x", *at->pdep, at->expected_pde); 761 762 if (success && verbose) { 763 if (at->expected_fault) { 764 printf("PASS (%x)\n", at->expected_error); 765 } else { 766 printf("PASS\n"); 767 } 768 } 769 return success; 770 } 771 772 static void ac_test_show(ac_test_t *at) 773 { 774 char line[5000]; 775 776 *line = 0; 777 strcat(line, "test"); 778 for (int i = 0; i < NR_AC_FLAGS; ++i) 779 if (at->flags & (1 << i)) { 780 strcat(line, " "); 781 strcat(line, ac_names[i]); 782 } 783 784 strcat(line, ": "); 785 printf("%s", line); 786 } 787 788 /* 789 * This test case is used to triger the bug which is fixed by 790 * commit e09e90a5 in the kvm tree 791 */ 792 static int corrupt_hugepage_triger(ac_pool_t *pool) 793 { 794 ac_test_t at1, at2; 795 796 ac_test_init(&at1, (void *)(0x123400000000)); 797 ac_test_init(&at2, (void *)(0x666600000000)); 798 799 at2.flags = AC_CPU_CR0_WP_MASK | AC_PDE_PSE_MASK | AC_PDE_PRESENT_MASK; 800 ac_test_setup_pte(&at2, pool); 801 if (!ac_test_do_access(&at2)) 802 goto err; 803 804 at1.flags = at2.flags | AC_PDE_WRITABLE_MASK; 805 ac_test_setup_pte(&at1, pool); 806 if (!ac_test_do_access(&at1)) 807 goto err; 808 809 at1.flags |= AC_ACCESS_WRITE_MASK; 810 ac_set_expected_status(&at1); 811 if (!ac_test_do_access(&at1)) 812 goto err; 813 814 at2.flags |= AC_ACCESS_WRITE_MASK; 815 ac_set_expected_status(&at2); 816 if (!ac_test_do_access(&at2)) 817 goto err; 818 819 return 1; 820 821 err: 822 printf("corrupt_hugepage_triger test fail\n"); 823 return 0; 824 } 825 826 /* 827 * This test case is used to triger the bug which is fixed by 828 * commit 3ddf6c06e13e in the kvm tree 829 */ 830 static int check_pfec_on_prefetch_pte(ac_pool_t *pool) 831 { 832 ac_test_t at1, at2; 833 834 ac_test_init(&at1, (void *)(0x123406001000)); 835 ac_test_init(&at2, (void *)(0x123406003000)); 836 837 at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK; 838 ac_setup_specific_pages(&at1, pool, 30 * 1024 * 1024, 30 * 1024 * 1024); 839 840 at2.flags = at1.flags | AC_PTE_NX_MASK; 841 ac_setup_specific_pages(&at2, pool, 30 * 1024 * 1024, 30 * 1024 * 1024); 842 843 if (!ac_test_do_access(&at1)) { 844 printf("%s: prepare fail\n", __FUNCTION__); 845 goto err; 846 } 847 848 if (!ac_test_do_access(&at2)) { 849 printf("%s: check PFEC on prefetch pte path fail\n", 850 __FUNCTION__); 851 goto err; 852 } 853 854 return 1; 855 856 err: 857 return 0; 858 } 859 860 /* 861 * If the write-fault access is from supervisor and CR0.WP is not set on the 862 * vcpu, kvm will fix it by adjusting pte access - it sets the W bit on pte 863 * and clears U bit. This is the chance that kvm can change pte access from 864 * readonly to writable. 865 * 866 * Unfortunately, the pte access is the access of 'direct' shadow page table, 867 * means direct sp.role.access = pte_access, then we will create a writable 868 * spte entry on the readonly shadow page table. It will cause Dirty bit is 869 * not tracked when two guest ptes point to the same large page. Note, it 870 * does not have other impact except Dirty bit since cr0.wp is encoded into 871 * sp.role. 872 * 873 * Note: to trigger this bug, hugepage should be disabled on host. 874 */ 875 static int check_large_pte_dirty_for_nowp(ac_pool_t *pool) 876 { 877 ac_test_t at1, at2; 878 879 ac_test_init(&at1, (void *)(0x123403000000)); 880 ac_test_init(&at2, (void *)(0x666606000000)); 881 882 at2.flags = AC_PDE_PRESENT_MASK | AC_PDE_PSE_MASK; 883 ac_test_setup_pte(&at2, pool); 884 if (!ac_test_do_access(&at2)) { 885 printf("%s: read on the first mapping fail.\n", __FUNCTION__); 886 goto err; 887 } 888 889 at1.flags = at2.flags | AC_ACCESS_WRITE_MASK; 890 ac_test_setup_pte(&at1, pool); 891 if (!ac_test_do_access(&at1)) { 892 printf("%s: write on the second mapping fail.\n", __FUNCTION__); 893 goto err; 894 } 895 896 at2.flags |= AC_ACCESS_WRITE_MASK; 897 ac_set_expected_status(&at2); 898 if (!ac_test_do_access(&at2)) { 899 printf("%s: write on the first mapping fail.\n", __FUNCTION__); 900 goto err; 901 } 902 903 return 1; 904 905 err: 906 return 0; 907 } 908 909 static int check_smep_andnot_wp(ac_pool_t *pool) 910 { 911 ac_test_t at1; 912 int err_prepare_andnot_wp, err_smep_andnot_wp; 913 914 if (!this_cpu_has(X86_FEATURE_SMEP)) { 915 return 1; 916 } 917 918 ac_test_init(&at1, (void *)(0x123406001000)); 919 920 at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK | 921 AC_PDE_USER_MASK | AC_PTE_USER_MASK | 922 AC_PDE_ACCESSED_MASK | AC_PTE_ACCESSED_MASK | 923 AC_CPU_CR4_SMEP_MASK | 924 AC_CPU_CR0_WP_MASK | 925 AC_ACCESS_WRITE_MASK; 926 ac_test_setup_pte(&at1, pool); 927 928 /* 929 * Here we write the ro user page when 930 * cr0.wp=0, then we execute it and SMEP 931 * fault should happen. 932 */ 933 err_prepare_andnot_wp = ac_test_do_access(&at1); 934 if (!err_prepare_andnot_wp) { 935 printf("%s: SMEP prepare fail\n", __FUNCTION__); 936 goto clean_up; 937 } 938 939 at1.flags &= ~AC_ACCESS_WRITE_MASK; 940 at1.flags |= AC_ACCESS_FETCH_MASK; 941 ac_set_expected_status(&at1); 942 err_smep_andnot_wp = ac_test_do_access(&at1); 943 944 clean_up: 945 set_cr4_smep(0); 946 947 if (!err_prepare_andnot_wp) 948 goto err; 949 if (!err_smep_andnot_wp) { 950 printf("%s: check SMEP without wp fail\n", __FUNCTION__); 951 goto err; 952 } 953 return 1; 954 955 err: 956 return 0; 957 } 958 959 static int check_effective_sp_permissions(ac_pool_t *pool) 960 { 961 unsigned long ptr1 = 0x123480000000; 962 unsigned long ptr2 = ptr1 + SZ_2M; 963 unsigned long ptr3 = ptr1 + SZ_1G; 964 unsigned long ptr4 = ptr3 + SZ_2M; 965 pt_element_t pmd = ac_test_alloc_pt(pool); 966 ac_test_t at1, at2, at3, at4; 967 int err_read_at1, err_write_at2; 968 int err_read_at3, err_write_at4; 969 970 /* 971 * pgd[] pud[] pmd[] virtual address pointers 972 * /->pmd1(u--)->pte1(uw-)->page1 <- ptr1 (u--) 973 * /->pud1(uw-)--->pmd2(uw-)->pte2(uw-)->page2 <- ptr2 (uw-) 974 * pgd-| (shared pmd[] as above) 975 * \->pud2(u--)--->pmd1(u--)->pte1(uw-)->page1 <- ptr3 (u--) 976 * \->pmd2(uw-)->pte2(uw-)->page2 <- ptr4 (u--) 977 * pud1 and pud2 point to the same pmd page. 978 */ 979 980 ac_test_init(&at1, (void *)(ptr1)); 981 at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK | 982 AC_PDE_USER_MASK | AC_PTE_USER_MASK | 983 AC_PDE_ACCESSED_MASK | AC_PTE_ACCESSED_MASK | 984 AC_PTE_WRITABLE_MASK | AC_ACCESS_USER_MASK; 985 __ac_setup_specific_pages(&at1, pool, false, pmd, 0); 986 987 ac_test_init(&at2, (void *)(ptr2)); 988 at2.flags = at1.flags | AC_PDE_WRITABLE_MASK | AC_PTE_DIRTY_MASK | AC_ACCESS_WRITE_MASK; 989 __ac_setup_specific_pages(&at2, pool, true, pmd, 0); 990 991 ac_test_init(&at3, (void *)(ptr3)); 992 at3.flags = AC_PDPTE_NO_WRITABLE_MASK | at1.flags; 993 __ac_setup_specific_pages(&at3, pool, true, pmd, 0); 994 995 ac_test_init(&at4, (void *)(ptr4)); 996 at4.flags = AC_PDPTE_NO_WRITABLE_MASK | at2.flags; 997 __ac_setup_specific_pages(&at4, pool, true, pmd, 0); 998 999 err_read_at1 = ac_test_do_access(&at1); 1000 if (!err_read_at1) { 1001 printf("%s: read access at1 fail\n", __FUNCTION__); 1002 return 0; 1003 } 1004 1005 err_write_at2 = ac_test_do_access(&at2); 1006 if (!err_write_at2) { 1007 printf("%s: write access at2 fail\n", __FUNCTION__); 1008 return 0; 1009 } 1010 1011 err_read_at3 = ac_test_do_access(&at3); 1012 if (!err_read_at3) { 1013 printf("%s: read access at3 fail\n", __FUNCTION__); 1014 return 0; 1015 } 1016 1017 err_write_at4 = ac_test_do_access(&at4); 1018 if (!err_write_at4) { 1019 printf("%s: write access at4 should fail\n", __FUNCTION__); 1020 return 0; 1021 } 1022 1023 return 1; 1024 } 1025 1026 static int ac_test_exec(ac_test_t *at, ac_pool_t *pool) 1027 { 1028 int r; 1029 1030 if (verbose) { 1031 ac_test_show(at); 1032 } 1033 ac_test_setup_pte(at, pool); 1034 r = ac_test_do_access(at); 1035 return r; 1036 } 1037 1038 typedef int (*ac_test_fn)(ac_pool_t *pool); 1039 const ac_test_fn ac_test_cases[] = 1040 { 1041 corrupt_hugepage_triger, 1042 check_pfec_on_prefetch_pte, 1043 check_large_pte_dirty_for_nowp, 1044 check_smep_andnot_wp, 1045 check_effective_sp_permissions, 1046 }; 1047 1048 static int ac_test_run(void) 1049 { 1050 ac_test_t at; 1051 ac_pool_t pool; 1052 int i, tests, successes; 1053 1054 printf("run\n"); 1055 tests = successes = 0; 1056 1057 shadow_cr0 = read_cr0(); 1058 shadow_cr4 = read_cr4(); 1059 shadow_efer = rdmsr(MSR_EFER); 1060 1061 if (cpuid_maxphyaddr() >= 52) { 1062 invalid_mask |= AC_PDE_BIT51_MASK; 1063 invalid_mask |= AC_PTE_BIT51_MASK; 1064 } 1065 if (cpuid_maxphyaddr() >= 37) { 1066 invalid_mask |= AC_PDE_BIT36_MASK; 1067 invalid_mask |= AC_PTE_BIT36_MASK; 1068 } 1069 1070 if (this_cpu_has(X86_FEATURE_PKU)) { 1071 set_cr4_pke(1); 1072 set_cr4_pke(0); 1073 /* Now PKRU = 0xFFFFFFFF. */ 1074 } else { 1075 tests++; 1076 if (write_cr4_checking(shadow_cr4 | X86_CR4_PKE) == GP_VECTOR) { 1077 successes++; 1078 invalid_mask |= AC_PKU_AD_MASK; 1079 invalid_mask |= AC_PKU_WD_MASK; 1080 invalid_mask |= AC_PKU_PKEY_MASK; 1081 invalid_mask |= AC_CPU_CR4_PKE_MASK; 1082 printf("CR4.PKE not available, disabling PKE tests\n"); 1083 } else { 1084 printf("Set PKE in CR4 - expect #GP: FAIL!\n"); 1085 set_cr4_pke(0); 1086 } 1087 } 1088 1089 if (!this_cpu_has(X86_FEATURE_SMEP)) { 1090 tests++; 1091 if (set_cr4_smep(1) == GP_VECTOR) { 1092 successes++; 1093 invalid_mask |= AC_CPU_CR4_SMEP_MASK; 1094 printf("CR4.SMEP not available, disabling SMEP tests\n"); 1095 } else { 1096 printf("Set SMEP in CR4 - expect #GP: FAIL!\n"); 1097 set_cr4_smep(0); 1098 } 1099 } 1100 1101 /* Toggling LA57 in 64-bit mode (guaranteed for this test) is illegal. */ 1102 if (this_cpu_has(X86_FEATURE_LA57)) { 1103 tests++; 1104 if (write_cr4_checking(shadow_cr4 ^ X86_CR4_LA57) == GP_VECTOR) 1105 successes++; 1106 1107 /* Force a VM-Exit on KVM, which doesn't intercept LA57 itself. */ 1108 tests++; 1109 if (write_cr4_checking(shadow_cr4 ^ (X86_CR4_LA57 | X86_CR4_PSE)) == GP_VECTOR) 1110 successes++; 1111 } 1112 1113 ac_env_int(&pool); 1114 ac_test_init(&at, (void *)(0x123400000000 + 16 * smp_id())); 1115 do { 1116 ++tests; 1117 successes += ac_test_exec(&at, &pool); 1118 } while (ac_test_bump(&at)); 1119 1120 for (i = 0; i < ARRAY_SIZE(ac_test_cases); i++) { 1121 ++tests; 1122 successes += ac_test_cases[i](&pool); 1123 } 1124 1125 printf("\n%d tests, %d failures\n", tests, tests - successes); 1126 1127 return successes == tests; 1128 } 1129 1130 int main(void) 1131 { 1132 int r; 1133 1134 printf("starting test\n\n"); 1135 page_table_levels = 4; 1136 r = ac_test_run(); 1137 1138 if (this_cpu_has(X86_FEATURE_LA57)) { 1139 page_table_levels = 5; 1140 printf("starting 5-level paging test.\n\n"); 1141 setup_5level_page_table(); 1142 r = ac_test_run(); 1143 } 1144 1145 return r ? 0 : 1; 1146 } 1147