17 #define PT_BASE_ADDR_MASK ((pt_element_t)((((pt_element_t)1 << 36) - 1) & PAGE_MASK))
31 	  (((address) >> (12 + ((level)-1) * 9)) & 511)
38  * Keep the PRESENT and reserved bits in the higher numbers so that they aren't
39  * toggled on every test, e.g. to keep entries in the TLB.
183 	/* 5-level paging, 1-based to avoid math. */
209 	for (i = at->pt_levels; i >= min_level; --i) {  in walk_va()
326 	pt_env->pt_pool_pa = AT_PAGING_STRUCTURES_PHYS;  in ac_env_int()
327 	pt_env->pt_pool_current = 0;  in ac_env_int()
328 	pt_env->pt_levels = page_table_levels;  in ac_env_int()
336 	 * Each test needs at most pt_levels-1 structures per virtual address,  in ac_test_alloc_pt()
339 	assert(pt_env->pt_pool_current < (4 * (pt_env->pt_levels - 1)));  in ac_test_alloc_pt()
341 	pt = pt_env->pt_pool_pa + (pt_env->pt_pool_current * PAGE_SIZE);  in ac_test_alloc_pt()
342 	pt_env->pt_pool_current++;  in ac_test_alloc_pt()
350 	unsigned long buddy_virt = buddy ? (unsigned long)buddy->virt : 0;  in __ac_test_init()
356 	 * top-level SPTE than the test, otherwise modifying SPTEs can affect  in __ac_test_init()
360 	assert(PT_INDEX(virt, pt_env->pt_levels) !=  in __ac_test_init()
361 	       PT_INDEX((unsigned long)__ac_test_init, pt_env->pt_levels));  in __ac_test_init()
365 	at->flags = 0;  in __ac_test_init()
366 	at->virt = (void *)virt;  in __ac_test_init()
367 	at->phys = AT_CODE_DATA_PHYS;  in __ac_test_init()
368 	at->pt_levels = pt_env->pt_levels;  in __ac_test_init()
370 	at->page_tables[0] = -1ull;  in __ac_test_init()
371 	at->page_tables[1] = -1ull;  in __ac_test_init()
374 	 * Zap the existing top-level PTE as it may be reused from a previous  in __ac_test_init()
375 	 * sub-test.  This allows runtime PTE modification to assert that two  in __ac_test_init()
378 	root_pt[PT_INDEX(virt, pt_env->pt_levels)] = 0;  in __ac_test_init()
380 	for (i = at->pt_levels; i > 1; i--) {  in __ac_test_init()
387 			at->page_tables[i] = buddy->page_tables[i];  in __ac_test_init()
389 			at->page_tables[i] = ac_test_alloc_pt(pt_env);  in __ac_test_init()
400 	at->flags = ((at->flags | invalid_mask) + 1) & ~invalid_mask;  in ac_test_bump_one()
401 	return at->flags < (1 << NR_AC_FLAGS);  in ac_test_bump_one()
408 	int flags = at->flags;  in ac_test_legal()
433 	 * pde.bit13 checks handling of reserved bits in largepage PDEs.  It is  in ac_test_legal()
453 	if (reserved & (reserved - 1))  in ac_test_legal()
478 		at->expected_fault = 1;  in ac_test_permissions()
481 		at->expected_fault = 1;  in ac_test_permissions()
484 		at->expected_fault = 1;  in ac_test_permissions()
487 		at->expected_fault = 1;  in ac_test_permissions()
491 			at->expected_fault = 1;  in ac_test_permissions()
492 			at->expected_error |= PFERR_PK_MASK;  in ac_test_permissions()
494 			at->expected_fault = 1;  in ac_test_permissions()
495 			at->expected_error |= PFERR_PK_MASK;  in ac_test_permissions()
499 	if (!at->expected_fault) {  in ac_test_permissions()
514 		at->expected_error |= PFERR_USER_MASK;  in ac_emulate_access()
517 		at->expected_error |= PFERR_WRITE_MASK;  in ac_emulate_access()
520 		at->expected_error |= PFERR_FETCH_MASK;  in ac_emulate_access()
523 		at->ignore_pde = PT_ACCESSED_MASK;  in ac_emulate_access()
530 		at->expected_fault = 1;  in ac_emulate_access()
532 			at->expected_error |= PFERR_RESERVED_MASK;  in ac_emulate_access()
534 			at->expected_error &= ~PFERR_PRESENT_MASK;  in ac_emulate_access()
544 		at->expected_pde |= ac_test_permissions(at, flags, writable,  in ac_emulate_access()
549 	at->expected_pde |= PT_ACCESSED_MASK;  in ac_emulate_access()
556 		at->expected_fault = 1;  in ac_emulate_access()
558 			at->expected_error |= PFERR_RESERVED_MASK;  in ac_emulate_access()
560 			at->expected_error &= ~PFERR_PRESENT_MASK;  in ac_emulate_access()
569 	at->expected_pte |= ac_test_permissions(at, flags, writable, user,  in ac_emulate_access()
574 	if (!at->expected_fault)  in ac_emulate_access()
575 		at->ignore_pde = 0;  in ac_emulate_access()
577 		at->expected_error &= ~PFERR_FETCH_MASK;  in ac_emulate_access()
583 		invlpg(at->virt);  in __ac_set_expected_status()
585 	if (at->ptep)  in __ac_set_expected_status()
586 		at->expected_pte = *at->ptep;  in __ac_set_expected_status()
587 	at->expected_pde = *at->pdep;  in __ac_set_expected_status()
588 	at->ignore_pde = 0;  in __ac_set_expected_status()
589 	at->expected_fault = 0;  in __ac_set_expected_status()
590 	at->expected_error = PFERR_PRESENT_MASK;  in __ac_set_expected_status()
592 	if (at->flags & AC_ACCESS_TWICE_MASK) {  in __ac_set_expected_status()
593 		ac_emulate_access(at, at->flags &  in __ac_set_expected_status()
597 		at->expected_fault = 0;  in __ac_set_expected_status()
598 		at->expected_error = PFERR_PRESENT_MASK;  in __ac_set_expected_status()
599 		at->ignore_pde = 0;  in __ac_set_expected_status()
602 	ac_emulate_access(at, at->flags);  in __ac_set_expected_status()
616 	    (pte & PT_BASE_ADDR_MASK) != at->page_tables[i]) {  in ac_get_pt()
618 			(unsigned long)at->virt, i,  in ac_get_pt()
619 			PT_INDEX((unsigned long)at->virt, i),  in ac_get_pt()
620 			pte, at->page_tables[i]);  in ac_get_pt()
628 	pte = at->page_tables[i] | (pte & (PT_DIRTY_MASK | PT_ACCESSED_MASK));  in ac_get_pt()
635 	int flags = at->flags;  in ac_test_setup_ptes()
638 	at->ptep = 0;  in ac_test_setup_ptes()
639 	for (i = at->pt_levels; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) {  in ac_test_setup_ptes()
641 		unsigned index = PT_INDEX((unsigned long)at->virt, i);  in ac_test_setup_ptes()
661 				/* The protection key is ignored on non-leaf entries.  */  in ac_test_setup_ptes()
665 				pte = at->phys & PT_PSE_BASE_ADDR_MASK;  in ac_test_setup_ptes()
688 			at->pdep = ptep;  in ac_test_setup_ptes()
691 			pte = at->phys & PT_BASE_ADDR_MASK;  in ac_test_setup_ptes()
710 			at->ptep = ptep;  in ac_test_setup_ptes()
726 	printf("------L%d I%lu: %lx\n", level, PT_INDEX(virt, level), *ptep);  in __dump_pte()
731 	unsigned long virt = (unsigned long)at->virt;  in dump_mapping()
732 	int flags = at->flags;  in dump_mapping()
734 	printf("Dump mapping: address: %p\n", at->virt);  in dump_mapping()
781 	int flags = at->flags;  in ac_test_do_access()
788 	*((unsigned char *)at->phys) = 0xc3; /* ret */  in ac_test_do_access()
810 			      : [addr]"r"(at->virt), [fep]"r"(F(AC_FEP))  in ac_test_do_access()
853 		      : [addr]"r"(at->virt),  in ac_test_do_access()
872 	ac_test_check(at, &success, fault && !at->expected_fault,  in ac_test_do_access()
874 	ac_test_check(at, &success, !fault && at->expected_fault,  in ac_test_do_access()
876 	ac_test_check(at, &success, fault && e != at->expected_error,  in ac_test_do_access()
877 		      "error code %x expected %x", e, at->expected_error);  in ac_test_do_access()
878 	if (at->ptep)  in ac_test_do_access()
879 		ac_test_check(at, &success, *at->ptep != at->expected_pte,  in ac_test_do_access()
880 			      "pte %x expected %x", *at->ptep, at->expected_pte);  in ac_test_do_access()
882 		      !pt_match(*at->pdep, at->expected_pde, at->ignore_pde),  in ac_test_do_access()
883 		      "pde %x expected %x", *at->pdep, at->expected_pde);  in ac_test_do_access()
886 		if (at->expected_fault) {  in ac_test_do_access()
887 			printf("PASS (%x)\n", at->expected_error);  in ac_test_do_access()
902 		if (at->flags & (1 << i)) {  in ac_test_show()
913  * commit e09e90a5 in the kvm tree
951  * commit 3ddf6c06e13e in the kvm tree
984  * If the write-fault access is from supervisor and CR0.WP is not set on the
985  * vcpu, kvm will fix it by adjusting pte access - it sets the W bit on pte
1091 	at->flags = TOGGLE_CR0_WP_TEST_BASE_FLAGS | flags;  in do_cr0_wp_access()
1096 	 * manually keep track of it where needed, e.g. in the guest page  in do_cr0_wp_access()
1101 	 * the current MMU context based on the soon-to-be-stale CR0.WP.  in do_cr0_wp_access()
1148 	 *                   /->pmd(u--)->pte1(uw-)->page1 <- ptr1 (u--)  in check_effective_sp_permissions()
1149 	 *      /->pud1(uw-)--->pmd(uw-)->pte2(uw-)->page2 <- ptr2 (uw-)  in check_effective_sp_permissions()
1150 	 * pgd-|  in check_effective_sp_permissions()
1151 	 *      \->pud2(u--)--->pmd(u--)->pte1(uw-)->page1 <- ptr3 (u--)  in check_effective_sp_permissions()
1152 	 *                   \->pmd(uw-)->pte2(uw-)->page2 <- ptr4 (u--)  in check_effective_sp_permissions()
1168 	/* Override the PMD (1-based index) to point at ptr1's PMD. */  in check_effective_sp_permissions()
1277 			printf("Set PKE in CR4 - expect #GP: FAIL!\n");  in ac_test_run()
1289 			printf("Set SMEP in CR4 - expect #GP: FAIL!\n");  in ac_test_run()
1294 	/* Toggling LA57 in 64-bit mode (guaranteed for this test) is illegal. */  in ac_test_run()
1300 		/* Force a VM-Exit on KVM, which doesn't intercept LA57 itself. */  in ac_test_run()
1318 	printf("\n%d tests, %d failures\n", tests, tests - successes);  in ac_test_run()
1320 	report(successes == tests, "%d-level paging tests%s", pt_levels,  in ac_test_run()