1 // SPDX-License-Identifier: GPL-2.0-only
12  *   Signed-off-by: Richard Fellner <richard.fellner@student.tugraz.at>
13  *   Signed-off-by: Moritz Lipp <moritz.lipp@iaik.tugraz.at>
14  *   Signed-off-by: Daniel Gruss <daniel.gruss@iaik.tugraz.at>
15  *   Signed-off-by: Michael Schwarz <michael.schwarz@iaik.tugraz.at>
43 #define pr_fmt(fmt)     "Kernel/User page tables isolation: " fmt
51  * Define the page-table levels we clone for user-space on 32
112 		return -EINVAL;  in pti_parse_cmdline()
132 	 * Top-level entries added to init_mm's usermode pgd after boot  in __pti_set_user_pgtbl()
139 	 * The user page tables get the full PGD, accessible from  in __pti_set_user_pgtbl()
142 	kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd;  in __pti_set_user_pgtbl()
145 	 * If this is normal user memory, make it NX in the kernel  in __pti_set_user_pgtbl()
148 	 * instead of allowing user code to execute with the wrong CR3.  in __pti_set_user_pgtbl()
151 	 *  - _PAGE_USER is not set.  This could be an executable  in __pti_set_user_pgtbl()
154 	 *  - we don't have NX support  in __pti_set_user_pgtbl()
155 	 *  - we're clearing the PGD (i.e. the new pgd is not present).  in __pti_set_user_pgtbl()
166  * Walk the user copy of the page tables (optionally) trying to allocate
177 		WARN_ONCE(1, "attempt to walk user address\n");  in pti_user_pagetable_walk_p4d()
194  * Walk the user copy of the page tables (optionally) trying to allocate
219 	/* The user page tables do not use large mappings: */  in pti_user_pagetable_walk_pmd()
240  * user/shadow page tables.  It is never used for userspace data.
270 		WARN_ONCE(1, "attempt to walk to user pte\n");  in pti_user_pagetable_walk_pte()
280 	unsigned int level;  in pti_setup_vsyscall()  local
282 	pte = lookup_address(VSYSCALL_ADDR, &level);  in pti_setup_vsyscall()
283 	if (!pte || WARN_ON(level != PG_LEVEL_4K) || pte_none(*pte))  in pti_setup_vsyscall()
304 		  enum pti_clone_level level)  in pti_clone_pgtable()  argument
344 		if (pmd_large(*pmd) || level == PTI_CLONE_PMD) {  in pti_clone_pgtable()
352 			 * called on well-known addresses anyway, so a non-  in pti_clone_pgtable()
360 			 * the user and kernel page tables.  It is effectively  in pti_clone_pgtable()
372 			 * tables will share the last-level page tables of this  in pti_clone_pgtable()
379 		} else if (level == PTI_CLONE_PTE) {  in pti_clone_pgtable()
381 			/* Walk the page-table down to the pte level */  in pti_clone_pgtable()
392 			/* Allocate PTE in the user page-table */  in pti_clone_pgtable()
414  * Clone a single p4d (i.e. a top-level entry on 4-level systems and a
415  * next-level entry on 5-level systems.
432  * Clone the CPU_ENTRY_AREA and associated data into the user space visible
468  * address space into the user page-tables, making PTI useless. So clone
469  * the page-table on the PMD level to prevent that.
483  * Clone the ESPFIX P4D into the user space visible page table
506  * to Meltdown-style attacks which make it trivial to find gadgets or
539 	 * data structures.  Keep the kernel image non-global in  in pti_kernel_image_global_ok()
550  * For some configurations, map all of kernel text into the user page
551  * tables.  This reduces TLB misses, especially on non-PCID systems.
567 	pr_debug("mapping partial kernel image into user address space\n");  in pti_clone_kernel_text()
579 	 * the last level for areas that are not huge-page-aligned.  in pti_clone_kernel_text()
582 	/* Set the global bit for normal non-__init kernel text: */  in pti_clone_kernel_text()
583 	set_memory_global(start, (end_global - start) >> PAGE_SHIFT);  in pti_clone_kernel_text()
602 	set_memory_nonglobal(start, (end - start) >> PAGE_SHIFT);  in pti_set_kernel_image_nonglobal()
617 	 * We check for X86_FEATURE_PCID here. But the init-code will  in pti_init()
628 		printk(KERN_WARNING "** You are using 32-bit PTI on a 64-bit PCID-capable CPU. **\n");  in pti_init()
630 		printk(KERN_WARNING "** switch to a 64-bit kernel!                             **\n");  in pti_init()
648  * Finalize the kernel mappings in the userspace page-table. Some of the
652  * userspace page-table.