xref: /kvm-unit-tests/x86/access.c (revision af928b22ada753b1ee34cad6ace3f2f6494991c6)
17d36db35SAvi Kivity 
27d36db35SAvi Kivity #include "libcflat.h"
3e7c37968SGleb Natapov #include "desc.h"
49c14b4d6SGleb Natapov #include "processor.h"
51ad15f10SAlexander Gordeev #include "asm/page.h"
67d36db35SAvi Kivity 
77d36db35SAvi Kivity #define smp_id() 0
87d36db35SAvi Kivity 
97d36db35SAvi Kivity #define true 1
107d36db35SAvi Kivity #define false 0
117d36db35SAvi Kivity 
127d36db35SAvi Kivity static _Bool verbose = false;
137d36db35SAvi Kivity 
147d36db35SAvi Kivity typedef unsigned long pt_element_t;
1544546719SPaolo Bonzini static int cpuid_7_ebx;
16e94079c5SPaolo Bonzini static int cpuid_7_ecx;
178acc274eSPaolo Bonzini static int invalid_mask;
18*af928b22SYu Zhang static int page_table_levels;
197d36db35SAvi Kivity 
207d36db35SAvi Kivity #define PT_BASE_ADDR_MASK ((pt_element_t)((((pt_element_t)1 << 40) - 1) & PAGE_MASK))
217d36db35SAvi Kivity #define PT_PSE_BASE_ADDR_MASK (PT_BASE_ADDR_MASK & ~(1ull << 21))
227d36db35SAvi Kivity 
237d36db35SAvi Kivity #define CR0_WP_MASK (1UL << 16)
24951181c8SYang, Wei #define CR4_SMEP_MASK (1UL << 20)
257d36db35SAvi Kivity 
267d36db35SAvi Kivity #define PFERR_PRESENT_MASK (1U << 0)
277d36db35SAvi Kivity #define PFERR_WRITE_MASK (1U << 1)
287d36db35SAvi Kivity #define PFERR_USER_MASK (1U << 2)
297d36db35SAvi Kivity #define PFERR_RESERVED_MASK (1U << 3)
307d36db35SAvi Kivity #define PFERR_FETCH_MASK (1U << 4)
31e94079c5SPaolo Bonzini #define PFERR_PK_MASK (1U << 5)
327d36db35SAvi Kivity 
337d36db35SAvi Kivity #define MSR_EFER 0xc0000080
347d36db35SAvi Kivity #define EFER_NX_MASK		(1ull << 11)
357d36db35SAvi Kivity 
36988c9e88SXiao Guangrong #define PT_INDEX(address, level)       \
37988c9e88SXiao Guangrong        ((address) >> (12 + ((level)-1) * 9)) & 511
38988c9e88SXiao Guangrong 
397d36db35SAvi Kivity /*
407d36db35SAvi Kivity  * page table access check tests
417d36db35SAvi Kivity  */
427d36db35SAvi Kivity 
437d36db35SAvi Kivity enum {
44f6a0180aSPaolo Bonzini     AC_PTE_PRESENT_BIT,
45f6a0180aSPaolo Bonzini     AC_PTE_WRITABLE_BIT,
46f6a0180aSPaolo Bonzini     AC_PTE_USER_BIT,
47f6a0180aSPaolo Bonzini     AC_PTE_ACCESSED_BIT,
48f6a0180aSPaolo Bonzini     AC_PTE_DIRTY_BIT,
49f6a0180aSPaolo Bonzini     AC_PTE_NX_BIT,
50f6a0180aSPaolo Bonzini     AC_PTE_BIT51_BIT,
517d36db35SAvi Kivity 
52f6a0180aSPaolo Bonzini     AC_PDE_PRESENT_BIT,
53f6a0180aSPaolo Bonzini     AC_PDE_WRITABLE_BIT,
54f6a0180aSPaolo Bonzini     AC_PDE_USER_BIT,
55f6a0180aSPaolo Bonzini     AC_PDE_ACCESSED_BIT,
56f6a0180aSPaolo Bonzini     AC_PDE_DIRTY_BIT,
57f6a0180aSPaolo Bonzini     AC_PDE_PSE_BIT,
58f6a0180aSPaolo Bonzini     AC_PDE_NX_BIT,
59f6a0180aSPaolo Bonzini     AC_PDE_BIT51_BIT,
60f6a0180aSPaolo Bonzini     AC_PDE_BIT13_BIT,
617d36db35SAvi Kivity 
62f6a0180aSPaolo Bonzini     AC_PKU_AD_BIT,
63f6a0180aSPaolo Bonzini     AC_PKU_WD_BIT,
64f6a0180aSPaolo Bonzini     AC_PKU_PKEY_BIT,
65e94079c5SPaolo Bonzini 
66f6a0180aSPaolo Bonzini     AC_ACCESS_USER_BIT,
67f6a0180aSPaolo Bonzini     AC_ACCESS_WRITE_BIT,
68f6a0180aSPaolo Bonzini     AC_ACCESS_FETCH_BIT,
69f6a0180aSPaolo Bonzini     AC_ACCESS_TWICE_BIT,
707d36db35SAvi Kivity 
71f6a0180aSPaolo Bonzini     AC_CPU_EFER_NX_BIT,
72f6a0180aSPaolo Bonzini     AC_CPU_CR0_WP_BIT,
73f6a0180aSPaolo Bonzini     AC_CPU_CR4_SMEP_BIT,
74f6a0180aSPaolo Bonzini     AC_CPU_CR4_PKE_BIT,
757d36db35SAvi Kivity 
767d36db35SAvi Kivity     NR_AC_FLAGS
777d36db35SAvi Kivity };
787d36db35SAvi Kivity 
79f6a0180aSPaolo Bonzini #define AC_PTE_PRESENT_MASK   (1 << AC_PTE_PRESENT_BIT)
80f6a0180aSPaolo Bonzini #define AC_PTE_WRITABLE_MASK  (1 << AC_PTE_WRITABLE_BIT)
81f6a0180aSPaolo Bonzini #define AC_PTE_USER_MASK      (1 << AC_PTE_USER_BIT)
82f6a0180aSPaolo Bonzini #define AC_PTE_ACCESSED_MASK  (1 << AC_PTE_ACCESSED_BIT)
83f6a0180aSPaolo Bonzini #define AC_PTE_DIRTY_MASK     (1 << AC_PTE_DIRTY_BIT)
84f6a0180aSPaolo Bonzini #define AC_PTE_NX_MASK        (1 << AC_PTE_NX_BIT)
85f6a0180aSPaolo Bonzini #define AC_PTE_BIT51_MASK     (1 << AC_PTE_BIT51_BIT)
86f6a0180aSPaolo Bonzini 
87f6a0180aSPaolo Bonzini #define AC_PDE_PRESENT_MASK   (1 << AC_PDE_PRESENT_BIT)
88f6a0180aSPaolo Bonzini #define AC_PDE_WRITABLE_MASK  (1 << AC_PDE_WRITABLE_BIT)
89f6a0180aSPaolo Bonzini #define AC_PDE_USER_MASK      (1 << AC_PDE_USER_BIT)
90f6a0180aSPaolo Bonzini #define AC_PDE_ACCESSED_MASK  (1 << AC_PDE_ACCESSED_BIT)
91f6a0180aSPaolo Bonzini #define AC_PDE_DIRTY_MASK     (1 << AC_PDE_DIRTY_BIT)
92f6a0180aSPaolo Bonzini #define AC_PDE_PSE_MASK       (1 << AC_PDE_PSE_BIT)
93f6a0180aSPaolo Bonzini #define AC_PDE_NX_MASK        (1 << AC_PDE_NX_BIT)
94f6a0180aSPaolo Bonzini #define AC_PDE_BIT51_MASK     (1 << AC_PDE_BIT51_BIT)
95f6a0180aSPaolo Bonzini #define AC_PDE_BIT13_MASK     (1 << AC_PDE_BIT13_BIT)
96f6a0180aSPaolo Bonzini 
97f6a0180aSPaolo Bonzini #define AC_PKU_AD_MASK        (1 << AC_PKU_AD_BIT)
98f6a0180aSPaolo Bonzini #define AC_PKU_WD_MASK        (1 << AC_PKU_WD_BIT)
99f6a0180aSPaolo Bonzini #define AC_PKU_PKEY_MASK      (1 << AC_PKU_PKEY_BIT)
100f6a0180aSPaolo Bonzini 
101f6a0180aSPaolo Bonzini #define AC_ACCESS_USER_MASK   (1 << AC_ACCESS_USER_BIT)
102f6a0180aSPaolo Bonzini #define AC_ACCESS_WRITE_MASK  (1 << AC_ACCESS_WRITE_BIT)
103f6a0180aSPaolo Bonzini #define AC_ACCESS_FETCH_MASK  (1 << AC_ACCESS_FETCH_BIT)
104f6a0180aSPaolo Bonzini #define AC_ACCESS_TWICE_MASK  (1 << AC_ACCESS_TWICE_BIT)
105f6a0180aSPaolo Bonzini 
106f6a0180aSPaolo Bonzini #define AC_CPU_EFER_NX_MASK   (1 << AC_CPU_EFER_NX_BIT)
107f6a0180aSPaolo Bonzini #define AC_CPU_CR0_WP_MASK    (1 << AC_CPU_CR0_WP_BIT)
108f6a0180aSPaolo Bonzini #define AC_CPU_CR4_SMEP_MASK  (1 << AC_CPU_CR4_SMEP_BIT)
109f6a0180aSPaolo Bonzini #define AC_CPU_CR4_PKE_MASK   (1 << AC_CPU_CR4_PKE_BIT)
110f6a0180aSPaolo Bonzini 
1117d36db35SAvi Kivity const char *ac_names[] = {
112f6a0180aSPaolo Bonzini     [AC_PTE_PRESENT_BIT] = "pte.p",
113f6a0180aSPaolo Bonzini     [AC_PTE_ACCESSED_BIT] = "pte.a",
114f6a0180aSPaolo Bonzini     [AC_PTE_WRITABLE_BIT] = "pte.rw",
115f6a0180aSPaolo Bonzini     [AC_PTE_USER_BIT] = "pte.user",
116f6a0180aSPaolo Bonzini     [AC_PTE_DIRTY_BIT] = "pte.d",
117f6a0180aSPaolo Bonzini     [AC_PTE_NX_BIT] = "pte.nx",
118f6a0180aSPaolo Bonzini     [AC_PTE_BIT51_BIT] = "pte.51",
119f6a0180aSPaolo Bonzini     [AC_PDE_PRESENT_BIT] = "pde.p",
120f6a0180aSPaolo Bonzini     [AC_PDE_ACCESSED_BIT] = "pde.a",
121f6a0180aSPaolo Bonzini     [AC_PDE_WRITABLE_BIT] = "pde.rw",
122f6a0180aSPaolo Bonzini     [AC_PDE_USER_BIT] = "pde.user",
123f6a0180aSPaolo Bonzini     [AC_PDE_DIRTY_BIT] = "pde.d",
124f6a0180aSPaolo Bonzini     [AC_PDE_PSE_BIT] = "pde.pse",
125f6a0180aSPaolo Bonzini     [AC_PDE_NX_BIT] = "pde.nx",
126f6a0180aSPaolo Bonzini     [AC_PDE_BIT51_BIT] = "pde.51",
127f6a0180aSPaolo Bonzini     [AC_PDE_BIT13_BIT] = "pde.13",
128f6a0180aSPaolo Bonzini     [AC_PKU_AD_BIT] = "pkru.ad",
129f6a0180aSPaolo Bonzini     [AC_PKU_WD_BIT] = "pkru.wd",
130f6a0180aSPaolo Bonzini     [AC_PKU_PKEY_BIT] = "pkey=1",
131f6a0180aSPaolo Bonzini     [AC_ACCESS_WRITE_BIT] = "write",
132f6a0180aSPaolo Bonzini     [AC_ACCESS_USER_BIT] = "user",
133f6a0180aSPaolo Bonzini     [AC_ACCESS_FETCH_BIT] = "fetch",
134f6a0180aSPaolo Bonzini     [AC_ACCESS_TWICE_BIT] = "twice",
135f6a0180aSPaolo Bonzini     [AC_CPU_EFER_NX_BIT] = "efer.nx",
136f6a0180aSPaolo Bonzini     [AC_CPU_CR0_WP_BIT] = "cr0.wp",
137f6a0180aSPaolo Bonzini     [AC_CPU_CR4_SMEP_BIT] = "cr4.smep",
138f6a0180aSPaolo Bonzini     [AC_CPU_CR4_PKE_BIT] = "cr4.pke",
1397d36db35SAvi Kivity };
1407d36db35SAvi Kivity 
1417d36db35SAvi Kivity static inline void *va(pt_element_t phys)
1427d36db35SAvi Kivity {
1437d36db35SAvi Kivity     return (void *)phys;
1447d36db35SAvi Kivity }
1457d36db35SAvi Kivity 
1467d36db35SAvi Kivity typedef struct {
1477d36db35SAvi Kivity     pt_element_t pt_pool;
1487d36db35SAvi Kivity     unsigned pt_pool_size;
1497d36db35SAvi Kivity     unsigned pt_pool_current;
1507d36db35SAvi Kivity } ac_pool_t;
1517d36db35SAvi Kivity 
1527d36db35SAvi Kivity typedef struct {
153f6a0180aSPaolo Bonzini     unsigned flags;
1547d36db35SAvi Kivity     void *virt;
1557d36db35SAvi Kivity     pt_element_t phys;
1567d36db35SAvi Kivity     pt_element_t *ptep;
1577d36db35SAvi Kivity     pt_element_t expected_pte;
1587d36db35SAvi Kivity     pt_element_t *pdep;
1597d36db35SAvi Kivity     pt_element_t expected_pde;
1607d36db35SAvi Kivity     pt_element_t ignore_pde;
1617d36db35SAvi Kivity     int expected_fault;
1627d36db35SAvi Kivity     unsigned expected_error;
1637d36db35SAvi Kivity } ac_test_t;
1647d36db35SAvi Kivity 
1657d36db35SAvi Kivity typedef struct {
1667d36db35SAvi Kivity     unsigned short limit;
1677d36db35SAvi Kivity     unsigned long linear_addr;
1687d36db35SAvi Kivity } __attribute__((packed)) descriptor_table_t;
1697d36db35SAvi Kivity 
1707d36db35SAvi Kivity 
1717d36db35SAvi Kivity static void ac_test_show(ac_test_t *at);
1727d36db35SAvi Kivity 
173951181c8SYang, Wei int write_cr4_checking(unsigned long val)
174951181c8SYang, Wei {
175951181c8SYang, Wei     asm volatile(ASM_TRY("1f")
176951181c8SYang, Wei             "mov %0,%%cr4\n\t"
177951181c8SYang, Wei             "1:": : "r" (val));
178951181c8SYang, Wei     return exception_vector();
179951181c8SYang, Wei }
180951181c8SYang, Wei 
1817d36db35SAvi Kivity void set_cr0_wp(int wp)
1827d36db35SAvi Kivity {
1837d36db35SAvi Kivity     unsigned long cr0 = read_cr0();
1848b7106c2SPaolo Bonzini     unsigned long old_cr0 = cr0;
1857d36db35SAvi Kivity 
1867d36db35SAvi Kivity     cr0 &= ~CR0_WP_MASK;
1877d36db35SAvi Kivity     if (wp)
1887d36db35SAvi Kivity 	cr0 |= CR0_WP_MASK;
1898b7106c2SPaolo Bonzini     if (old_cr0 != cr0)
1907d36db35SAvi Kivity         write_cr0(cr0);
1917d36db35SAvi Kivity }
1927d36db35SAvi Kivity 
19310318f98SPaolo Bonzini unsigned set_cr4_smep(int smep)
194951181c8SYang, Wei {
195951181c8SYang, Wei     unsigned long cr4 = read_cr4();
1968b7106c2SPaolo Bonzini     unsigned long old_cr4 = cr4;
1972835a6b1SPaolo Bonzini     extern u64 ptl2[];
19810318f98SPaolo Bonzini     unsigned r;
199951181c8SYang, Wei 
200951181c8SYang, Wei     cr4 &= ~CR4_SMEP_MASK;
201951181c8SYang, Wei     if (smep)
202951181c8SYang, Wei 	cr4 |= CR4_SMEP_MASK;
2038b7106c2SPaolo Bonzini     if (old_cr4 == cr4)
20410318f98SPaolo Bonzini         return 0;
2058b7106c2SPaolo Bonzini 
2068b7106c2SPaolo Bonzini     if (smep)
2078b7106c2SPaolo Bonzini         ptl2[2] &= ~PT_USER_MASK;
20810318f98SPaolo Bonzini     r = write_cr4_checking(cr4);
2092835a6b1SPaolo Bonzini     if (!smep)
2102835a6b1SPaolo Bonzini         ptl2[2] |= PT_USER_MASK;
21110318f98SPaolo Bonzini     return r;
212951181c8SYang, Wei }
213951181c8SYang, Wei 
214e94079c5SPaolo Bonzini void set_cr4_pke(int pke)
215e94079c5SPaolo Bonzini {
216e94079c5SPaolo Bonzini     unsigned long cr4 = read_cr4();
2178b7106c2SPaolo Bonzini     unsigned long old_cr4 = cr4;
2188b7106c2SPaolo Bonzini 
2198b7106c2SPaolo Bonzini     cr4 &= ~X86_CR4_PKE;
2208b7106c2SPaolo Bonzini     if (pke)
2218b7106c2SPaolo Bonzini 	cr4 |= X86_CR4_PKE;
2228b7106c2SPaolo Bonzini     if (old_cr4 == cr4)
2238b7106c2SPaolo Bonzini         return;
224e94079c5SPaolo Bonzini 
225e94079c5SPaolo Bonzini     /* Check that protection keys do not affect accesses when CR4.PKE=0.  */
226e94079c5SPaolo Bonzini     if ((read_cr4() & X86_CR4_PKE) && !pke) {
227cb2755dbSPaolo Bonzini         write_pkru(0xfffffffc);
228e94079c5SPaolo Bonzini     }
229e94079c5SPaolo Bonzini     write_cr4(cr4);
230e94079c5SPaolo Bonzini }
231e94079c5SPaolo Bonzini 
2327d36db35SAvi Kivity void set_efer_nx(int nx)
2337d36db35SAvi Kivity {
2348b7106c2SPaolo Bonzini     unsigned long long efer = rdmsr(MSR_EFER);
2358b7106c2SPaolo Bonzini     unsigned long long old_efer = efer;
2367d36db35SAvi Kivity 
2377d36db35SAvi Kivity     efer &= ~EFER_NX_MASK;
2387d36db35SAvi Kivity     if (nx)
2397d36db35SAvi Kivity 	efer |= EFER_NX_MASK;
2408b7106c2SPaolo Bonzini     if (old_efer != efer)
2417d36db35SAvi Kivity         wrmsr(MSR_EFER, efer);
2427d36db35SAvi Kivity }
2437d36db35SAvi Kivity 
2447d36db35SAvi Kivity static void ac_env_int(ac_pool_t *pool)
2457d36db35SAvi Kivity {
2467d36db35SAvi Kivity     extern char page_fault, kernel_entry;
2479c14b4d6SGleb Natapov     set_idt_entry(14, &page_fault, 0);
2489c14b4d6SGleb Natapov     set_idt_entry(0x20, &kernel_entry, 3);
2497d36db35SAvi Kivity 
2507d36db35SAvi Kivity     pool->pt_pool = 33 * 1024 * 1024;
2517d36db35SAvi Kivity     pool->pt_pool_size = 120 * 1024 * 1024 - pool->pt_pool;
2527d36db35SAvi Kivity     pool->pt_pool_current = 0;
2537d36db35SAvi Kivity }
2547d36db35SAvi Kivity 
2557d36db35SAvi Kivity void ac_test_init(ac_test_t *at, void *virt)
2567d36db35SAvi Kivity {
2577d36db35SAvi Kivity     wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NX_MASK);
2587d36db35SAvi Kivity     set_cr0_wp(1);
259f6a0180aSPaolo Bonzini     at->flags = 0;
2607d36db35SAvi Kivity     at->virt = virt;
2617d36db35SAvi Kivity     at->phys = 32 * 1024 * 1024;
2627d36db35SAvi Kivity }
2637d36db35SAvi Kivity 
2647d36db35SAvi Kivity int ac_test_bump_one(ac_test_t *at)
2657d36db35SAvi Kivity {
2668acc274eSPaolo Bonzini     at->flags = ((at->flags | invalid_mask) + 1) & ~invalid_mask;
267f6a0180aSPaolo Bonzini     return at->flags < (1 << NR_AC_FLAGS);
2687d36db35SAvi Kivity }
2697d36db35SAvi Kivity 
270875c1b49SPaolo Bonzini #define F(x)  ((flags & x##_MASK) != 0)
271f6a0180aSPaolo Bonzini 
2727d36db35SAvi Kivity _Bool ac_test_legal(ac_test_t *at)
2737d36db35SAvi Kivity {
274875c1b49SPaolo Bonzini     int flags = at->flags;
275875c1b49SPaolo Bonzini 
276f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_FETCH) && F(AC_ACCESS_WRITE))
27703298f10SPaolo Bonzini 	return false;
27803298f10SPaolo Bonzini 
279951181c8SYang, Wei     /*
280951181c8SYang, Wei      * Since we convert current page to kernel page when cr4.smep=1,
281951181c8SYang, Wei      * we can't switch to user mode.
282951181c8SYang, Wei      */
283f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_USER) && F(AC_CPU_CR4_SMEP))
2847d36db35SAvi Kivity 	return false;
28503298f10SPaolo Bonzini 
28603298f10SPaolo Bonzini     /*
287e94079c5SPaolo Bonzini      * Only test protection key faults if CR4.PKE=1.
288e94079c5SPaolo Bonzini      */
289f6a0180aSPaolo Bonzini     if (!F(AC_CPU_CR4_PKE) &&
290f6a0180aSPaolo Bonzini         (F(AC_PKU_AD) || F(AC_PKU_WD))) {
291e94079c5SPaolo Bonzini 	return false;
292e94079c5SPaolo Bonzini     }
293e94079c5SPaolo Bonzini 
294e94079c5SPaolo Bonzini     /*
29503298f10SPaolo Bonzini      * pde.bit13 checks handling of reserved bits in largepage PDEs.  It is
29603298f10SPaolo Bonzini      * meaningless if there is a PTE.
29703298f10SPaolo Bonzini      */
298f6a0180aSPaolo Bonzini     if (!F(AC_PDE_PSE) && F(AC_PDE_BIT13))
29903298f10SPaolo Bonzini         return false;
30003298f10SPaolo Bonzini 
3017d36db35SAvi Kivity     return true;
3027d36db35SAvi Kivity }
3037d36db35SAvi Kivity 
3047d36db35SAvi Kivity int ac_test_bump(ac_test_t *at)
3057d36db35SAvi Kivity {
3067d36db35SAvi Kivity     int ret;
3077d36db35SAvi Kivity 
3087d36db35SAvi Kivity     ret = ac_test_bump_one(at);
3097d36db35SAvi Kivity     while (ret && !ac_test_legal(at))
3107d36db35SAvi Kivity 	ret = ac_test_bump_one(at);
3117d36db35SAvi Kivity     return ret;
3127d36db35SAvi Kivity }
3137d36db35SAvi Kivity 
3147d36db35SAvi Kivity pt_element_t ac_test_alloc_pt(ac_pool_t *pool)
3157d36db35SAvi Kivity {
3167d36db35SAvi Kivity     pt_element_t ret = pool->pt_pool + pool->pt_pool_current;
3177d36db35SAvi Kivity     pool->pt_pool_current += PAGE_SIZE;
3187d36db35SAvi Kivity     return ret;
3197d36db35SAvi Kivity }
3207d36db35SAvi Kivity 
3217d36db35SAvi Kivity _Bool ac_test_enough_room(ac_pool_t *pool)
3227d36db35SAvi Kivity {
323*af928b22SYu Zhang     return pool->pt_pool_current + 5 * PAGE_SIZE <= pool->pt_pool_size;
3247d36db35SAvi Kivity }
3257d36db35SAvi Kivity 
3267d36db35SAvi Kivity void ac_test_reset_pt_pool(ac_pool_t *pool)
3277d36db35SAvi Kivity {
3287d36db35SAvi Kivity     pool->pt_pool_current = 0;
3297d36db35SAvi Kivity }
3307d36db35SAvi Kivity 
33157465a25SPaolo Bonzini pt_element_t ac_test_permissions(ac_test_t *at, unsigned flags, bool writable,
3324615aeb5SPaolo Bonzini                                  bool user, bool executable)
33357465a25SPaolo Bonzini {
33457465a25SPaolo Bonzini     bool kwritable = !F(AC_CPU_CR0_WP) && !F(AC_ACCESS_USER);
33557465a25SPaolo Bonzini     pt_element_t expected = 0;
33657465a25SPaolo Bonzini 
33757465a25SPaolo Bonzini     if (F(AC_ACCESS_USER) && !user)
33857465a25SPaolo Bonzini 	at->expected_fault = 1;
33957465a25SPaolo Bonzini 
34057465a25SPaolo Bonzini     if (F(AC_ACCESS_WRITE) && !writable && !kwritable)
34157465a25SPaolo Bonzini 	at->expected_fault = 1;
34257465a25SPaolo Bonzini 
34357465a25SPaolo Bonzini     if (F(AC_ACCESS_FETCH) && !executable)
34457465a25SPaolo Bonzini 	at->expected_fault = 1;
34557465a25SPaolo Bonzini 
34657465a25SPaolo Bonzini     if (F(AC_ACCESS_FETCH) && user && F(AC_CPU_CR4_SMEP))
34757465a25SPaolo Bonzini         at->expected_fault = 1;
34857465a25SPaolo Bonzini 
349fcd86f8fSPaolo Bonzini     if (user && !F(AC_ACCESS_FETCH) && F(AC_PKU_PKEY) && F(AC_CPU_CR4_PKE)) {
35057465a25SPaolo Bonzini         if (F(AC_PKU_AD)) {
35157465a25SPaolo Bonzini             at->expected_fault = 1;
35257465a25SPaolo Bonzini             at->expected_error |= PFERR_PK_MASK;
35357465a25SPaolo Bonzini         } else if (F(AC_ACCESS_WRITE) && F(AC_PKU_WD) && !kwritable) {
35457465a25SPaolo Bonzini             at->expected_fault = 1;
35557465a25SPaolo Bonzini             at->expected_error |= PFERR_PK_MASK;
35657465a25SPaolo Bonzini         }
35757465a25SPaolo Bonzini     }
35857465a25SPaolo Bonzini 
3594615aeb5SPaolo Bonzini     if (!at->expected_fault) {
36076631c0bSPaolo Bonzini         expected |= PT_ACCESSED_MASK;
3614615aeb5SPaolo Bonzini         if (F(AC_ACCESS_WRITE))
3624615aeb5SPaolo Bonzini             expected |= PT_DIRTY_MASK;
3634615aeb5SPaolo Bonzini     }
36476631c0bSPaolo Bonzini 
36557465a25SPaolo Bonzini     return expected;
36657465a25SPaolo Bonzini }
36757465a25SPaolo Bonzini 
368413a2ac1SPaolo Bonzini void ac_emulate_access(ac_test_t *at, unsigned flags)
3697d36db35SAvi Kivity {
37007fc5026SPaolo Bonzini     bool pde_valid, pte_valid;
37157465a25SPaolo Bonzini     bool user, writable, executable;
3727d36db35SAvi Kivity 
373f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_USER))
3747d36db35SAvi Kivity 	at->expected_error |= PFERR_USER_MASK;
3757d36db35SAvi Kivity 
376f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_WRITE))
3777d36db35SAvi Kivity 	at->expected_error |= PFERR_WRITE_MASK;
3787d36db35SAvi Kivity 
379f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_FETCH))
3807d36db35SAvi Kivity 	at->expected_error |= PFERR_FETCH_MASK;
3817d36db35SAvi Kivity 
3821e814344SPaolo Bonzini     if (!F(AC_PDE_ACCESSED))
3831e814344SPaolo Bonzini         at->ignore_pde = PT_ACCESSED_MASK;
3841e814344SPaolo Bonzini 
38507fc5026SPaolo Bonzini     pde_valid = F(AC_PDE_PRESENT)
38607fc5026SPaolo Bonzini         && !F(AC_PDE_BIT51) && !F(AC_PDE_BIT13)
38707fc5026SPaolo Bonzini         && !(F(AC_PDE_NX) && !F(AC_CPU_EFER_NX));
38807fc5026SPaolo Bonzini 
38907fc5026SPaolo Bonzini     if (!pde_valid) {
39007fc5026SPaolo Bonzini         at->expected_fault = 1;
39107fc5026SPaolo Bonzini 	if (F(AC_PDE_PRESENT)) {
39207fc5026SPaolo Bonzini             at->expected_error |= PFERR_RESERVED_MASK;
39307fc5026SPaolo Bonzini         } else {
39407fc5026SPaolo Bonzini             at->expected_error &= ~PFERR_PRESENT_MASK;
39507fc5026SPaolo Bonzini         }
3961e814344SPaolo Bonzini 	goto fault;
39707fc5026SPaolo Bonzini     }
3981e814344SPaolo Bonzini 
399d91c7adbSPaolo Bonzini     writable = F(AC_PDE_WRITABLE);
400d91c7adbSPaolo Bonzini     user = F(AC_PDE_USER);
401d91c7adbSPaolo Bonzini     executable = !F(AC_PDE_NX);
402d91c7adbSPaolo Bonzini 
403f6a0180aSPaolo Bonzini     if (F(AC_PDE_PSE)) {
4044615aeb5SPaolo Bonzini         at->expected_pde |= ac_test_permissions(at, flags, writable, user,
4054615aeb5SPaolo Bonzini                                                 executable);
4067d36db35SAvi Kivity 	goto no_pte;
4077d36db35SAvi Kivity     }
4087d36db35SAvi Kivity 
4094615aeb5SPaolo Bonzini     at->expected_pde |= PT_ACCESSED_MASK;
4104615aeb5SPaolo Bonzini 
41107fc5026SPaolo Bonzini     pte_valid = F(AC_PTE_PRESENT)
41207fc5026SPaolo Bonzini         && !F(AC_PTE_BIT51)
41307fc5026SPaolo Bonzini         && !(F(AC_PTE_NX) && !F(AC_CPU_EFER_NX));
4147d36db35SAvi Kivity 
41507fc5026SPaolo Bonzini     if (!pte_valid) {
41607fc5026SPaolo Bonzini         at->expected_fault = 1;
41707fc5026SPaolo Bonzini 	if (F(AC_PTE_PRESENT)) {
41807fc5026SPaolo Bonzini             at->expected_error |= PFERR_RESERVED_MASK;
41907fc5026SPaolo Bonzini         } else {
42007fc5026SPaolo Bonzini             at->expected_error &= ~PFERR_PRESENT_MASK;
42107fc5026SPaolo Bonzini         }
422e94079c5SPaolo Bonzini 	goto fault;
42307fc5026SPaolo Bonzini     }
424e94079c5SPaolo Bonzini 
425d91c7adbSPaolo Bonzini     writable &= F(AC_PTE_WRITABLE);
426d91c7adbSPaolo Bonzini     user &= F(AC_PTE_USER);
427d91c7adbSPaolo Bonzini     executable &= !F(AC_PTE_NX);
428d91c7adbSPaolo Bonzini 
42957465a25SPaolo Bonzini     at->expected_pte |= ac_test_permissions(at, flags, writable, user,
4304615aeb5SPaolo Bonzini                                             executable);
4317d36db35SAvi Kivity 
4327d36db35SAvi Kivity no_pte:
4337d36db35SAvi Kivity fault:
4347d36db35SAvi Kivity     if (!at->expected_fault)
4357d36db35SAvi Kivity         at->ignore_pde = 0;
436f6a0180aSPaolo Bonzini     if (!F(AC_CPU_EFER_NX) && !F(AC_CPU_CR4_SMEP))
4377d36db35SAvi Kivity         at->expected_error &= ~PFERR_FETCH_MASK;
4387d36db35SAvi Kivity }
4397d36db35SAvi Kivity 
440413a2ac1SPaolo Bonzini void ac_set_expected_status(ac_test_t *at)
441413a2ac1SPaolo Bonzini {
442413a2ac1SPaolo Bonzini     invlpg(at->virt);
443413a2ac1SPaolo Bonzini 
444413a2ac1SPaolo Bonzini     if (at->ptep)
445413a2ac1SPaolo Bonzini 	at->expected_pte = *at->ptep;
446413a2ac1SPaolo Bonzini     at->expected_pde = *at->pdep;
447413a2ac1SPaolo Bonzini     at->ignore_pde = 0;
448413a2ac1SPaolo Bonzini     at->expected_fault = 0;
449413a2ac1SPaolo Bonzini     at->expected_error = PFERR_PRESENT_MASK;
450413a2ac1SPaolo Bonzini 
451413a2ac1SPaolo Bonzini     if (at->flags & AC_ACCESS_TWICE_MASK) {
452413a2ac1SPaolo Bonzini         ac_emulate_access(at, at->flags & ~AC_ACCESS_WRITE_MASK
453413a2ac1SPaolo Bonzini                           & ~AC_ACCESS_FETCH_MASK & ~AC_ACCESS_USER_MASK);
454413a2ac1SPaolo Bonzini         at->expected_fault = 0;
455413a2ac1SPaolo Bonzini 	at->expected_error = PFERR_PRESENT_MASK;
456413a2ac1SPaolo Bonzini         at->ignore_pde = 0;
457413a2ac1SPaolo Bonzini     }
458413a2ac1SPaolo Bonzini 
459413a2ac1SPaolo Bonzini     ac_emulate_access(at, at->flags);
460413a2ac1SPaolo Bonzini }
461413a2ac1SPaolo Bonzini 
462aca2f8c5SXiao Guangrong void __ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool, u64 pd_page,
463aca2f8c5SXiao Guangrong 			       u64 pt_page)
464aca2f8c5SXiao Guangrong 
4657d36db35SAvi Kivity {
4667d36db35SAvi Kivity     unsigned long root = read_cr3();
467875c1b49SPaolo Bonzini     int flags = at->flags;
468*af928b22SYu Zhang     bool skip = true;
4697d36db35SAvi Kivity 
4707d36db35SAvi Kivity     if (!ac_test_enough_room(pool))
4717d36db35SAvi Kivity 	ac_test_reset_pt_pool(pool);
4727d36db35SAvi Kivity 
4737d36db35SAvi Kivity     at->ptep = 0;
474*af928b22SYu Zhang     for (int i = page_table_levels; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) {
4757d36db35SAvi Kivity 	pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK);
476988c9e88SXiao Guangrong 	unsigned index = PT_INDEX((unsigned long)at->virt, i);
4777d36db35SAvi Kivity 	pt_element_t pte = 0;
478*af928b22SYu Zhang 
479*af928b22SYu Zhang 	/*
480*af928b22SYu Zhang 	 * Reuse existing page tables along the path to the test code and data
481*af928b22SYu Zhang 	 * (which is in the bottom 2MB).
482*af928b22SYu Zhang 	 */
483*af928b22SYu Zhang 	if (skip && i >= 2 && index == 0) {
484*af928b22SYu Zhang 	    goto next;
485*af928b22SYu Zhang 	}
486*af928b22SYu Zhang 	skip = false;
487*af928b22SYu Zhang 
4887d36db35SAvi Kivity 	switch (i) {
489*af928b22SYu Zhang 	case 5:
4907d36db35SAvi Kivity 	case 4:
4917d36db35SAvi Kivity 	case 3:
492aca2f8c5SXiao Guangrong 	    pte = pd_page ? pd_page : ac_test_alloc_pt(pool);
493aca2f8c5SXiao Guangrong 	    pte |= PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
4947d36db35SAvi Kivity 	    break;
4957d36db35SAvi Kivity 	case 2:
496f6a0180aSPaolo Bonzini 	    if (!F(AC_PDE_PSE)) {
497aca2f8c5SXiao Guangrong 		pte = pt_page ? pt_page : ac_test_alloc_pt(pool);
498e94079c5SPaolo Bonzini 		/* The protection key is ignored on non-leaf entries.  */
499f6a0180aSPaolo Bonzini                 if (F(AC_PKU_PKEY))
500e94079c5SPaolo Bonzini                     pte |= 2ull << 59;
501e94079c5SPaolo Bonzini 	    } else {
5027d36db35SAvi Kivity 		pte = at->phys & PT_PSE_BASE_ADDR_MASK;
503d10d16e1SAlexander Gordeev 		pte |= PT_PAGE_SIZE_MASK;
504f6a0180aSPaolo Bonzini                 if (F(AC_PKU_PKEY))
505e94079c5SPaolo Bonzini                     pte |= 1ull << 59;
5067d36db35SAvi Kivity 	    }
507f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_PRESENT))
5087d36db35SAvi Kivity 		pte |= PT_PRESENT_MASK;
509f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_WRITABLE))
5107d36db35SAvi Kivity 		pte |= PT_WRITABLE_MASK;
511f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_USER))
5127d36db35SAvi Kivity 		pte |= PT_USER_MASK;
513f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_ACCESSED))
5147d36db35SAvi Kivity 		pte |= PT_ACCESSED_MASK;
515f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_DIRTY))
5167d36db35SAvi Kivity 		pte |= PT_DIRTY_MASK;
517f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_NX))
518d10d16e1SAlexander Gordeev 		pte |= PT64_NX_MASK;
519f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_BIT51))
5207d36db35SAvi Kivity 		pte |= 1ull << 51;
521f6a0180aSPaolo Bonzini 	    if (F(AC_PDE_BIT13))
52203298f10SPaolo Bonzini 		pte |= 1ull << 13;
5237d36db35SAvi Kivity 	    at->pdep = &vroot[index];
5247d36db35SAvi Kivity 	    break;
5257d36db35SAvi Kivity 	case 1:
5267d36db35SAvi Kivity 	    pte = at->phys & PT_BASE_ADDR_MASK;
527f6a0180aSPaolo Bonzini 	    if (F(AC_PKU_PKEY))
528e94079c5SPaolo Bonzini 		pte |= 1ull << 59;
529f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_PRESENT))
5307d36db35SAvi Kivity 		pte |= PT_PRESENT_MASK;
531f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_WRITABLE))
5327d36db35SAvi Kivity 		pte |= PT_WRITABLE_MASK;
533f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_USER))
5347d36db35SAvi Kivity 		pte |= PT_USER_MASK;
535f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_ACCESSED))
5367d36db35SAvi Kivity 		pte |= PT_ACCESSED_MASK;
537f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_DIRTY))
5387d36db35SAvi Kivity 		pte |= PT_DIRTY_MASK;
539f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_NX))
540d10d16e1SAlexander Gordeev 		pte |= PT64_NX_MASK;
541f6a0180aSPaolo Bonzini 	    if (F(AC_PTE_BIT51))
5427d36db35SAvi Kivity 		pte |= 1ull << 51;
5437d36db35SAvi Kivity 	    at->ptep = &vroot[index];
5447d36db35SAvi Kivity 	    break;
5457d36db35SAvi Kivity 	}
5467d36db35SAvi Kivity 	vroot[index] = pte;
547*af928b22SYu Zhang  next:
5487d36db35SAvi Kivity 	root = vroot[index];
5497d36db35SAvi Kivity     }
5507d36db35SAvi Kivity     ac_set_expected_status(at);
5517d36db35SAvi Kivity }
5527d36db35SAvi Kivity 
553aca2f8c5SXiao Guangrong static void ac_test_setup_pte(ac_test_t *at, ac_pool_t *pool)
554aca2f8c5SXiao Guangrong {
555aca2f8c5SXiao Guangrong 	__ac_setup_specific_pages(at, pool, 0, 0);
556aca2f8c5SXiao Guangrong }
557aca2f8c5SXiao Guangrong 
558aca2f8c5SXiao Guangrong static void ac_setup_specific_pages(ac_test_t *at, ac_pool_t *pool,
559aca2f8c5SXiao Guangrong 				    u64 pd_page, u64 pt_page)
560aca2f8c5SXiao Guangrong {
561aca2f8c5SXiao Guangrong 	return __ac_setup_specific_pages(at, pool, pd_page, pt_page);
562aca2f8c5SXiao Guangrong }
563aca2f8c5SXiao Guangrong 
564988c9e88SXiao Guangrong static void dump_mapping(ac_test_t *at)
565988c9e88SXiao Guangrong {
566988c9e88SXiao Guangrong 	unsigned long root = read_cr3();
567875c1b49SPaolo Bonzini         int flags = at->flags;
568988c9e88SXiao Guangrong 	int i;
569988c9e88SXiao Guangrong 
570b006d7ebSAndrew Jones 	printf("Dump mapping: address: %p\n", at->virt);
571*af928b22SYu Zhang 	for (i = page_table_levels ; i >= 1 && (i >= 2 || !F(AC_PDE_PSE)); --i) {
572988c9e88SXiao Guangrong 		pt_element_t *vroot = va(root & PT_BASE_ADDR_MASK);
573988c9e88SXiao Guangrong 		unsigned index = PT_INDEX((unsigned long)at->virt, i);
574988c9e88SXiao Guangrong 		pt_element_t pte = vroot[index];
575988c9e88SXiao Guangrong 
576b006d7ebSAndrew Jones 		printf("------L%d: %lx\n", i, pte);
577988c9e88SXiao Guangrong 		root = vroot[index];
578988c9e88SXiao Guangrong 	}
579988c9e88SXiao Guangrong }
580988c9e88SXiao Guangrong 
5817d36db35SAvi Kivity static void ac_test_check(ac_test_t *at, _Bool *success_ret, _Bool cond,
5827d36db35SAvi Kivity                           const char *fmt, ...)
5837d36db35SAvi Kivity {
5847d36db35SAvi Kivity     va_list ap;
5857d36db35SAvi Kivity     char buf[500];
5867d36db35SAvi Kivity 
5877d36db35SAvi Kivity     if (!*success_ret) {
5887d36db35SAvi Kivity         return;
5897d36db35SAvi Kivity     }
5907d36db35SAvi Kivity 
5917d36db35SAvi Kivity     if (!cond) {
5927d36db35SAvi Kivity         return;
5937d36db35SAvi Kivity     }
5947d36db35SAvi Kivity 
5957d36db35SAvi Kivity     *success_ret = false;
5967d36db35SAvi Kivity 
5977d36db35SAvi Kivity     if (!verbose) {
5982500efdeSPaolo Bonzini         puts("\n");
5997d36db35SAvi Kivity         ac_test_show(at);
6007d36db35SAvi Kivity     }
6017d36db35SAvi Kivity 
6027d36db35SAvi Kivity     va_start(ap, fmt);
6037d36db35SAvi Kivity     vsnprintf(buf, sizeof(buf), fmt, ap);
6047d36db35SAvi Kivity     va_end(ap);
6057d36db35SAvi Kivity     printf("FAIL: %s\n", buf);
606988c9e88SXiao Guangrong     dump_mapping(at);
6077d36db35SAvi Kivity }
6087d36db35SAvi Kivity 
6097d36db35SAvi Kivity static int pt_match(pt_element_t pte1, pt_element_t pte2, pt_element_t ignore)
6107d36db35SAvi Kivity {
6117d36db35SAvi Kivity     pte1 &= ~ignore;
6127d36db35SAvi Kivity     pte2 &= ~ignore;
6137d36db35SAvi Kivity     return pte1 == pte2;
6147d36db35SAvi Kivity }
6157d36db35SAvi Kivity 
6167d36db35SAvi Kivity int ac_test_do_access(ac_test_t *at)
6177d36db35SAvi Kivity {
6187d36db35SAvi Kivity     static unsigned unique = 42;
6197d36db35SAvi Kivity     int fault = 0;
6207d36db35SAvi Kivity     unsigned e;
6217d36db35SAvi Kivity     static unsigned char user_stack[4096];
6227d36db35SAvi Kivity     unsigned long rsp;
6237d36db35SAvi Kivity     _Bool success = true;
624875c1b49SPaolo Bonzini     int flags = at->flags;
6257d36db35SAvi Kivity 
6267d36db35SAvi Kivity     ++unique;
6272500efdeSPaolo Bonzini     if (!(unique & 65535)) {
6282500efdeSPaolo Bonzini         puts(".");
6292500efdeSPaolo Bonzini     }
6307d36db35SAvi Kivity 
6317d36db35SAvi Kivity     *((unsigned char *)at->phys) = 0xc3; /* ret */
6327d36db35SAvi Kivity 
6337d36db35SAvi Kivity     unsigned r = unique;
634f6a0180aSPaolo Bonzini     set_cr0_wp(F(AC_CPU_CR0_WP));
635f6a0180aSPaolo Bonzini     set_efer_nx(F(AC_CPU_EFER_NX));
636f6a0180aSPaolo Bonzini     set_cr4_pke(F(AC_CPU_CR4_PKE));
637f6a0180aSPaolo Bonzini     if (F(AC_CPU_CR4_PKE)) {
638f6a0180aSPaolo Bonzini         /* WD2=AD2=1, WD1=F(AC_PKU_WD), AD1=F(AC_PKU_AD) */
639f6a0180aSPaolo Bonzini         write_pkru(0x30 | (F(AC_PKU_WD) ? 8 : 0) |
640f6a0180aSPaolo Bonzini                    (F(AC_PKU_AD) ? 4 : 0));
641e94079c5SPaolo Bonzini     }
642e94079c5SPaolo Bonzini 
643f6a0180aSPaolo Bonzini     set_cr4_smep(F(AC_CPU_CR4_SMEP));
6447d36db35SAvi Kivity 
645f6a0180aSPaolo Bonzini     if (F(AC_ACCESS_TWICE)) {
6467d36db35SAvi Kivity 	asm volatile (
6477d36db35SAvi Kivity 	    "mov $fixed2, %%rsi \n\t"
6487d36db35SAvi Kivity 	    "mov (%[addr]), %[reg] \n\t"
6497d36db35SAvi Kivity 	    "fixed2:"
6507d36db35SAvi Kivity 	    : [reg]"=r"(r), [fault]"=a"(fault), "=b"(e)
6517d36db35SAvi Kivity 	    : [addr]"r"(at->virt)
6527d36db35SAvi Kivity 	    : "rsi"
6537d36db35SAvi Kivity 	    );
6547d36db35SAvi Kivity 	fault = 0;
6557d36db35SAvi Kivity     }
6567d36db35SAvi Kivity 
6577d36db35SAvi Kivity     asm volatile ("mov $fixed1, %%rsi \n\t"
6587d36db35SAvi Kivity 		  "mov %%rsp, %%rdx \n\t"
6597d36db35SAvi Kivity 		  "cmp $0, %[user] \n\t"
6607d36db35SAvi Kivity 		  "jz do_access \n\t"
6617d36db35SAvi Kivity 		  "push %%rax; mov %[user_ds], %%ax; mov %%ax, %%ds; pop %%rax  \n\t"
6627d36db35SAvi Kivity 		  "pushq %[user_ds] \n\t"
6637d36db35SAvi Kivity 		  "pushq %[user_stack_top] \n\t"
6647d36db35SAvi Kivity 		  "pushfq \n\t"
6657d36db35SAvi Kivity 		  "pushq %[user_cs] \n\t"
6667d36db35SAvi Kivity 		  "pushq $do_access \n\t"
6677d36db35SAvi Kivity 		  "iretq \n"
6687d36db35SAvi Kivity 		  "do_access: \n\t"
6697d36db35SAvi Kivity 		  "cmp $0, %[fetch] \n\t"
6707d36db35SAvi Kivity 		  "jnz 2f \n\t"
6717d36db35SAvi Kivity 		  "cmp $0, %[write] \n\t"
6727d36db35SAvi Kivity 		  "jnz 1f \n\t"
6737d36db35SAvi Kivity 		  "mov (%[addr]), %[reg] \n\t"
6747d36db35SAvi Kivity 		  "jmp done \n\t"
6757d36db35SAvi Kivity 		  "1: mov %[reg], (%[addr]) \n\t"
6767d36db35SAvi Kivity 		  "jmp done \n\t"
6777d36db35SAvi Kivity 		  "2: call *%[addr] \n\t"
6787d36db35SAvi Kivity 		  "done: \n"
6797d36db35SAvi Kivity 		  "fixed1: \n"
6807d36db35SAvi Kivity 		  "int %[kernel_entry_vector] \n\t"
6817d36db35SAvi Kivity 		  "back_to_kernel:"
6827d36db35SAvi Kivity 		  : [reg]"+r"(r), "+a"(fault), "=b"(e), "=&d"(rsp)
6837d36db35SAvi Kivity 		  : [addr]"r"(at->virt),
684f6a0180aSPaolo Bonzini 		    [write]"r"(F(AC_ACCESS_WRITE)),
685f6a0180aSPaolo Bonzini 		    [user]"r"(F(AC_ACCESS_USER)),
686f6a0180aSPaolo Bonzini 		    [fetch]"r"(F(AC_ACCESS_FETCH)),
68769d8fe0eSPaolo Bonzini 		    [user_ds]"i"(USER_DS),
68869d8fe0eSPaolo Bonzini 		    [user_cs]"i"(USER_CS),
6897d36db35SAvi Kivity 		    [user_stack_top]"r"(user_stack + sizeof user_stack),
6907d36db35SAvi Kivity 		    [kernel_entry_vector]"i"(0x20)
6917d36db35SAvi Kivity 		  : "rsi");
6927d36db35SAvi Kivity 
6937d36db35SAvi Kivity     asm volatile (".section .text.pf \n\t"
6947d36db35SAvi Kivity 		  "page_fault: \n\t"
6957d36db35SAvi Kivity 		  "pop %rbx \n\t"
6967d36db35SAvi Kivity 		  "mov %rsi, (%rsp) \n\t"
6977d36db35SAvi Kivity 		  "movl $1, %eax \n\t"
6987d36db35SAvi Kivity 		  "iretq \n\t"
6997d36db35SAvi Kivity 		  ".section .text");
7007d36db35SAvi Kivity 
7017d36db35SAvi Kivity     asm volatile (".section .text.entry \n\t"
7027d36db35SAvi Kivity 		  "kernel_entry: \n\t"
7037d36db35SAvi Kivity 		  "mov %rdx, %rsp \n\t"
7047d36db35SAvi Kivity 		  "jmp back_to_kernel \n\t"
7057d36db35SAvi Kivity 		  ".section .text");
7067d36db35SAvi Kivity 
7077d36db35SAvi Kivity     ac_test_check(at, &success, fault && !at->expected_fault,
7087d36db35SAvi Kivity                   "unexpected fault");
7097d36db35SAvi Kivity     ac_test_check(at, &success, !fault && at->expected_fault,
7107d36db35SAvi Kivity                   "unexpected access");
7117d36db35SAvi Kivity     ac_test_check(at, &success, fault && e != at->expected_error,
7127d36db35SAvi Kivity                   "error code %x expected %x", e, at->expected_error);
7137d36db35SAvi Kivity     ac_test_check(at, &success, at->ptep && *at->ptep != at->expected_pte,
7147d36db35SAvi Kivity                   "pte %x expected %x", *at->ptep, at->expected_pte);
7157d36db35SAvi Kivity     ac_test_check(at, &success,
7167d36db35SAvi Kivity                   !pt_match(*at->pdep, at->expected_pde, at->ignore_pde),
7177d36db35SAvi Kivity                   "pde %x expected %x", *at->pdep, at->expected_pde);
7187d36db35SAvi Kivity 
7197d36db35SAvi Kivity     if (success && verbose) {
7203ed8b0adSPaolo Bonzini 	if (at->expected_fault) {
7213ed8b0adSPaolo Bonzini             printf("PASS (%x)\n", at->expected_error);
7223ed8b0adSPaolo Bonzini 	} else {
7237d36db35SAvi Kivity             printf("PASS\n");
7247d36db35SAvi Kivity 	}
7253ed8b0adSPaolo Bonzini     }
7267d36db35SAvi Kivity     return success;
7277d36db35SAvi Kivity }
7287d36db35SAvi Kivity 
7297d36db35SAvi Kivity static void ac_test_show(ac_test_t *at)
7307d36db35SAvi Kivity {
7317d36db35SAvi Kivity     char line[5000];
7327d36db35SAvi Kivity 
7337d36db35SAvi Kivity     *line = 0;
7347d36db35SAvi Kivity     strcat(line, "test");
7357d36db35SAvi Kivity     for (int i = 0; i < NR_AC_FLAGS; ++i)
736f6a0180aSPaolo Bonzini 	if (at->flags & (1 << i)) {
7377d36db35SAvi Kivity 	    strcat(line, " ");
7387d36db35SAvi Kivity 	    strcat(line, ac_names[i]);
7397d36db35SAvi Kivity 	}
7407d36db35SAvi Kivity     strcat(line, ": ");
7417d36db35SAvi Kivity     printf("%s", line);
7427d36db35SAvi Kivity }
7437d36db35SAvi Kivity 
7447d36db35SAvi Kivity /*
7457d36db35SAvi Kivity  * This test case is used to triger the bug which is fixed by
7467d36db35SAvi Kivity  * commit e09e90a5 in the kvm tree
7477d36db35SAvi Kivity  */
7487d36db35SAvi Kivity static int corrupt_hugepage_triger(ac_pool_t *pool)
7497d36db35SAvi Kivity {
7507d36db35SAvi Kivity     ac_test_t at1, at2;
7517d36db35SAvi Kivity 
7527d36db35SAvi Kivity     ac_test_init(&at1, (void *)(0x123400000000));
7537d36db35SAvi Kivity     ac_test_init(&at2, (void *)(0x666600000000));
7547d36db35SAvi Kivity 
755f6a0180aSPaolo Bonzini     at2.flags = AC_CPU_CR0_WP_MASK | AC_PDE_PSE_MASK | AC_PDE_PRESENT_MASK;
7567d36db35SAvi Kivity     ac_test_setup_pte(&at2, pool);
7577d36db35SAvi Kivity     if (!ac_test_do_access(&at2))
7587d36db35SAvi Kivity         goto err;
7597d36db35SAvi Kivity 
760f6a0180aSPaolo Bonzini     at1.flags = at2.flags | AC_PDE_WRITABLE_MASK;
7617d36db35SAvi Kivity     ac_test_setup_pte(&at1, pool);
7627d36db35SAvi Kivity     if (!ac_test_do_access(&at1))
7637d36db35SAvi Kivity         goto err;
7647d36db35SAvi Kivity 
765f6a0180aSPaolo Bonzini     at1.flags |= AC_ACCESS_WRITE_MASK;
7667d36db35SAvi Kivity     ac_set_expected_status(&at1);
7677d36db35SAvi Kivity     if (!ac_test_do_access(&at1))
7687d36db35SAvi Kivity         goto err;
7697d36db35SAvi Kivity 
770f6a0180aSPaolo Bonzini     at2.flags |= AC_ACCESS_WRITE_MASK;
7717d36db35SAvi Kivity     ac_set_expected_status(&at2);
7727d36db35SAvi Kivity     if (!ac_test_do_access(&at2))
7737d36db35SAvi Kivity         goto err;
7747d36db35SAvi Kivity 
7757d36db35SAvi Kivity     return 1;
7767d36db35SAvi Kivity 
7777d36db35SAvi Kivity err:
7787d36db35SAvi Kivity     printf("corrupt_hugepage_triger test fail\n");
7797d36db35SAvi Kivity     return 0;
7807d36db35SAvi Kivity }
7817d36db35SAvi Kivity 
782aca2f8c5SXiao Guangrong /*
783aca2f8c5SXiao Guangrong  * This test case is used to triger the bug which is fixed by
784aca2f8c5SXiao Guangrong  * commit 3ddf6c06e13e in the kvm tree
785aca2f8c5SXiao Guangrong  */
786aca2f8c5SXiao Guangrong static int check_pfec_on_prefetch_pte(ac_pool_t *pool)
787aca2f8c5SXiao Guangrong {
788aca2f8c5SXiao Guangrong 	ac_test_t at1, at2;
789aca2f8c5SXiao Guangrong 
790aca2f8c5SXiao Guangrong 	ac_test_init(&at1, (void *)(0x123406001000));
791aca2f8c5SXiao Guangrong 	ac_test_init(&at2, (void *)(0x123406003000));
792aca2f8c5SXiao Guangrong 
793f6a0180aSPaolo Bonzini 	at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK;
794aca2f8c5SXiao Guangrong 	ac_setup_specific_pages(&at1, pool, 30 * 1024 * 1024, 30 * 1024 * 1024);
795aca2f8c5SXiao Guangrong 
796f6a0180aSPaolo Bonzini         at2.flags = at1.flags | AC_PTE_NX_MASK;
797aca2f8c5SXiao Guangrong 	ac_setup_specific_pages(&at2, pool, 30 * 1024 * 1024, 30 * 1024 * 1024);
798aca2f8c5SXiao Guangrong 
799aca2f8c5SXiao Guangrong 	if (!ac_test_do_access(&at1)) {
800aca2f8c5SXiao Guangrong 		printf("%s: prepare fail\n", __FUNCTION__);
801aca2f8c5SXiao Guangrong 		goto err;
802aca2f8c5SXiao Guangrong 	}
803aca2f8c5SXiao Guangrong 
804aca2f8c5SXiao Guangrong 	if (!ac_test_do_access(&at2)) {
805aca2f8c5SXiao Guangrong 		printf("%s: check PFEC on prefetch pte path fail\n",
806aca2f8c5SXiao Guangrong 			__FUNCTION__);
807aca2f8c5SXiao Guangrong 		goto err;
808aca2f8c5SXiao Guangrong 	}
809aca2f8c5SXiao Guangrong 
810aca2f8c5SXiao Guangrong 	return 1;
811aca2f8c5SXiao Guangrong 
812aca2f8c5SXiao Guangrong err:
813aca2f8c5SXiao Guangrong     return 0;
814aca2f8c5SXiao Guangrong }
815aca2f8c5SXiao Guangrong 
816fcead254SXiao Guangrong /*
817fcead254SXiao Guangrong  * If the write-fault access is from supervisor and CR0.WP is not set on the
818fcead254SXiao Guangrong  * vcpu, kvm will fix it by adjusting pte access - it sets the W bit on pte
819fcead254SXiao Guangrong  * and clears U bit. This is the chance that kvm can change pte access from
820fcead254SXiao Guangrong  * readonly to writable.
821fcead254SXiao Guangrong  *
822fcead254SXiao Guangrong  * Unfortunately, the pte access is the access of 'direct' shadow page table,
823fcead254SXiao Guangrong  * means direct sp.role.access = pte_access, then we will create a writable
824fcead254SXiao Guangrong  * spte entry on the readonly shadow page table. It will cause Dirty bit is
825fcead254SXiao Guangrong  * not tracked when two guest ptes point to the same large page. Note, it
826fcead254SXiao Guangrong  * does not have other impact except Dirty bit since cr0.wp is encoded into
827fcead254SXiao Guangrong  * sp.role.
828fcead254SXiao Guangrong  *
829fcead254SXiao Guangrong  * Note: to trigger this bug, hugepage should be disabled on host.
830fcead254SXiao Guangrong  */
831fcead254SXiao Guangrong static int check_large_pte_dirty_for_nowp(ac_pool_t *pool)
832fcead254SXiao Guangrong {
833fcead254SXiao Guangrong 	ac_test_t at1, at2;
834fcead254SXiao Guangrong 
835fcead254SXiao Guangrong 	ac_test_init(&at1, (void *)(0x123403000000));
836fcead254SXiao Guangrong 	ac_test_init(&at2, (void *)(0x666606000000));
837fcead254SXiao Guangrong 
838f6a0180aSPaolo Bonzini         at2.flags = AC_PDE_PRESENT_MASK | AC_PDE_PSE_MASK;
839fcead254SXiao Guangrong 	ac_test_setup_pte(&at2, pool);
840fcead254SXiao Guangrong 	if (!ac_test_do_access(&at2)) {
841fcead254SXiao Guangrong 		printf("%s: read on the first mapping fail.\n", __FUNCTION__);
842fcead254SXiao Guangrong 		goto err;
843fcead254SXiao Guangrong 	}
844fcead254SXiao Guangrong 
845f6a0180aSPaolo Bonzini         at1.flags = at2.flags | AC_ACCESS_WRITE_MASK;
846fcead254SXiao Guangrong 	ac_test_setup_pte(&at1, pool);
847fcead254SXiao Guangrong 	if (!ac_test_do_access(&at1)) {
848fcead254SXiao Guangrong 		printf("%s: write on the second mapping fail.\n", __FUNCTION__);
849fcead254SXiao Guangrong 		goto err;
850fcead254SXiao Guangrong 	}
851fcead254SXiao Guangrong 
852f6a0180aSPaolo Bonzini 	at2.flags |= AC_ACCESS_WRITE_MASK;
853fcead254SXiao Guangrong 	ac_set_expected_status(&at2);
854fcead254SXiao Guangrong 	if (!ac_test_do_access(&at2)) {
855fcead254SXiao Guangrong 		printf("%s: write on the first mapping fail.\n", __FUNCTION__);
856fcead254SXiao Guangrong 		goto err;
857fcead254SXiao Guangrong 	}
858fcead254SXiao Guangrong 
859fcead254SXiao Guangrong 	return 1;
860fcead254SXiao Guangrong 
861fcead254SXiao Guangrong err:
862fcead254SXiao Guangrong 	return 0;
863fcead254SXiao Guangrong }
864fcead254SXiao Guangrong 
865951181c8SYang, Wei static int check_smep_andnot_wp(ac_pool_t *pool)
866951181c8SYang, Wei {
867951181c8SYang, Wei 	ac_test_t at1;
868951181c8SYang, Wei 	int err_prepare_andnot_wp, err_smep_andnot_wp;
869951181c8SYang, Wei 
8708acc274eSPaolo Bonzini 	if (!(cpuid_7_ebx & (1 << 7))) {
8718acc274eSPaolo Bonzini 	    return 1;
8728acc274eSPaolo Bonzini 	}
8738acc274eSPaolo Bonzini 
874951181c8SYang, Wei 	ac_test_init(&at1, (void *)(0x123406001000));
875951181c8SYang, Wei 
876f6a0180aSPaolo Bonzini 	at1.flags = AC_PDE_PRESENT_MASK | AC_PTE_PRESENT_MASK |
877f6a0180aSPaolo Bonzini             AC_PDE_USER_MASK | AC_PTE_USER_MASK |
878f6a0180aSPaolo Bonzini             AC_PDE_ACCESSED_MASK | AC_PTE_ACCESSED_MASK |
879f6a0180aSPaolo Bonzini             AC_CPU_CR4_SMEP_MASK |
880f6a0180aSPaolo Bonzini             AC_CPU_CR0_WP_MASK |
881f6a0180aSPaolo Bonzini             AC_ACCESS_WRITE_MASK;
882951181c8SYang, Wei 	ac_test_setup_pte(&at1, pool);
883951181c8SYang, Wei 
884951181c8SYang, Wei 	/*
885951181c8SYang, Wei 	 * Here we write the ro user page when
886951181c8SYang, Wei 	 * cr0.wp=0, then we execute it and SMEP
887951181c8SYang, Wei 	 * fault should happen.
888951181c8SYang, Wei 	 */
889951181c8SYang, Wei 	err_prepare_andnot_wp = ac_test_do_access(&at1);
890951181c8SYang, Wei 	if (!err_prepare_andnot_wp) {
891951181c8SYang, Wei 		printf("%s: SMEP prepare fail\n", __FUNCTION__);
892951181c8SYang, Wei 		goto clean_up;
893951181c8SYang, Wei 	}
894951181c8SYang, Wei 
895f6a0180aSPaolo Bonzini         at1.flags &= ~AC_ACCESS_WRITE_MASK;
896f6a0180aSPaolo Bonzini         at1.flags |= AC_ACCESS_FETCH_MASK;
897951181c8SYang, Wei         ac_set_expected_status(&at1);
898951181c8SYang, Wei         err_smep_andnot_wp = ac_test_do_access(&at1);
899951181c8SYang, Wei 
900951181c8SYang, Wei clean_up:
901951181c8SYang, Wei 	set_cr4_smep(0);
902951181c8SYang, Wei 
903951181c8SYang, Wei 	if (!err_prepare_andnot_wp)
904951181c8SYang, Wei 		goto err;
905951181c8SYang, Wei 	if (!err_smep_andnot_wp) {
906951181c8SYang, Wei 		printf("%s: check SMEP without wp fail\n", __FUNCTION__);
907951181c8SYang, Wei 		goto err;
908951181c8SYang, Wei 	}
909951181c8SYang, Wei 	return 1;
910951181c8SYang, Wei 
911951181c8SYang, Wei err:
912951181c8SYang, Wei 	return 0;
913951181c8SYang, Wei }
914951181c8SYang, Wei 
9157d36db35SAvi Kivity int ac_test_exec(ac_test_t *at, ac_pool_t *pool)
9167d36db35SAvi Kivity {
9177d36db35SAvi Kivity     int r;
9187d36db35SAvi Kivity 
9197d36db35SAvi Kivity     if (verbose) {
9207d36db35SAvi Kivity         ac_test_show(at);
9217d36db35SAvi Kivity     }
9227d36db35SAvi Kivity     ac_test_setup_pte(at, pool);
9237d36db35SAvi Kivity     r = ac_test_do_access(at);
9247d36db35SAvi Kivity     return r;
9257d36db35SAvi Kivity }
9267d36db35SAvi Kivity 
927aca2f8c5SXiao Guangrong typedef int (*ac_test_fn)(ac_pool_t *pool);
928aca2f8c5SXiao Guangrong const ac_test_fn ac_test_cases[] =
929aca2f8c5SXiao Guangrong {
930aca2f8c5SXiao Guangrong 	corrupt_hugepage_triger,
931aca2f8c5SXiao Guangrong 	check_pfec_on_prefetch_pte,
932fcead254SXiao Guangrong 	check_large_pte_dirty_for_nowp,
933951181c8SYang, Wei 	check_smep_andnot_wp
934aca2f8c5SXiao Guangrong };
935aca2f8c5SXiao Guangrong 
9367d36db35SAvi Kivity int ac_test_run(void)
9377d36db35SAvi Kivity {
9387d36db35SAvi Kivity     ac_test_t at;
9397d36db35SAvi Kivity     ac_pool_t pool;
940aca2f8c5SXiao Guangrong     int i, tests, successes;
9417d36db35SAvi Kivity 
9427d36db35SAvi Kivity     printf("run\n");
9437d36db35SAvi Kivity     tests = successes = 0;
9448acc274eSPaolo Bonzini 
9458acc274eSPaolo Bonzini     if (cpuid_7_ecx & (1 << 3)) {
9468acc274eSPaolo Bonzini         set_cr4_pke(1);
9478acc274eSPaolo Bonzini         set_cr4_pke(0);
9488acc274eSPaolo Bonzini         /* Now PKRU = 0xFFFFFFFF.  */
9498acc274eSPaolo Bonzini     } else {
9508acc274eSPaolo Bonzini 	unsigned long cr4 = read_cr4();
9518acc274eSPaolo Bonzini 	tests++;
9528acc274eSPaolo Bonzini 	if (write_cr4_checking(cr4 | X86_CR4_PKE) == GP_VECTOR) {
9538acc274eSPaolo Bonzini             successes++;
9548acc274eSPaolo Bonzini             invalid_mask |= AC_PKU_AD_MASK;
9558acc274eSPaolo Bonzini             invalid_mask |= AC_PKU_WD_MASK;
9568acc274eSPaolo Bonzini             invalid_mask |= AC_PKU_PKEY_MASK;
9578acc274eSPaolo Bonzini             invalid_mask |= AC_CPU_CR4_PKE_MASK;
9588acc274eSPaolo Bonzini             printf("CR4.PKE not available, disabling PKE tests\n");
9598acc274eSPaolo Bonzini 	} else {
9608acc274eSPaolo Bonzini             printf("Set PKE in CR4 - expect #GP: FAIL!\n");
9618acc274eSPaolo Bonzini             set_cr4_pke(0);
9628acc274eSPaolo Bonzini 	}
9638acc274eSPaolo Bonzini     }
9648acc274eSPaolo Bonzini 
9658acc274eSPaolo Bonzini     if (!(cpuid_7_ebx & (1 << 7))) {
9668acc274eSPaolo Bonzini 	tests++;
96710318f98SPaolo Bonzini 	if (set_cr4_smep(1) == GP_VECTOR) {
9688acc274eSPaolo Bonzini             successes++;
9698acc274eSPaolo Bonzini             invalid_mask |= AC_CPU_CR4_SMEP_MASK;
9708acc274eSPaolo Bonzini             printf("CR4.SMEP not available, disabling SMEP tests\n");
9718acc274eSPaolo Bonzini 	} else {
9728acc274eSPaolo Bonzini             printf("Set SMEP in CR4 - expect #GP: FAIL!\n");
9738acc274eSPaolo Bonzini             set_cr4_smep(0);
9748acc274eSPaolo Bonzini 	}
9758acc274eSPaolo Bonzini     }
9768acc274eSPaolo Bonzini 
9777d36db35SAvi Kivity     ac_env_int(&pool);
9787d36db35SAvi Kivity     ac_test_init(&at, (void *)(0x123400000000 + 16 * smp_id()));
9797d36db35SAvi Kivity     do {
9807d36db35SAvi Kivity 	++tests;
9817d36db35SAvi Kivity 	successes += ac_test_exec(&at, &pool);
9827d36db35SAvi Kivity     } while (ac_test_bump(&at));
9837d36db35SAvi Kivity 
984aca2f8c5SXiao Guangrong     for (i = 0; i < ARRAY_SIZE(ac_test_cases); i++) {
9857d36db35SAvi Kivity 	++tests;
986aca2f8c5SXiao Guangrong 	successes += ac_test_cases[i](&pool);
987aca2f8c5SXiao Guangrong     }
9887d36db35SAvi Kivity 
9897d36db35SAvi Kivity     printf("\n%d tests, %d failures\n", tests, tests - successes);
9907d36db35SAvi Kivity 
9917d36db35SAvi Kivity     return successes == tests;
9927d36db35SAvi Kivity }
9937d36db35SAvi Kivity 
9947d36db35SAvi Kivity int main()
9957d36db35SAvi Kivity {
9967d36db35SAvi Kivity     int r;
9977d36db35SAvi Kivity 
9988acc274eSPaolo Bonzini     setup_idt();
9998acc274eSPaolo Bonzini 
100044546719SPaolo Bonzini     cpuid_7_ebx = cpuid(7).b;
1001e94079c5SPaolo Bonzini     cpuid_7_ecx = cpuid(7).c;
1002e94079c5SPaolo Bonzini 
10037d36db35SAvi Kivity     printf("starting test\n\n");
1004*af928b22SYu Zhang     page_table_levels = 4;
10057d36db35SAvi Kivity     r = ac_test_run();
1006*af928b22SYu Zhang 
1007*af928b22SYu Zhang     if (cpuid_7_ecx & (1 << 16)) {
1008*af928b22SYu Zhang         page_table_levels = 5;
1009*af928b22SYu Zhang         setup_5level_page_table();
1010*af928b22SYu Zhang         printf("starting 5-level paging test.\n\n");
1011*af928b22SYu Zhang         r = ac_test_run();
1012*af928b22SYu Zhang     }
1013*af928b22SYu Zhang 
10147d36db35SAvi Kivity     return r ? 0 : 1;
10157d36db35SAvi Kivity }
1016