1 // SPDX-License-Identifier: GPL-2.0-only
15 #include <linux/dma-direct.h>
21 #include <linux/dma-mapping.h>
31 #include <asm/processor-flags.h>
49 /* Buffer used for early in-place encryption by BSP, no locking needed */
53  * SNP-specific routine which needs to additionally change the page state from
64 		 * @paddr needs to be accessed decrypted, mark the page shared in  in snp_memcpy()
71 		/* Restore the page state after the memcpy. */  in snp_memcpy()
75 		 * @paddr need to be accessed encrypted, no need for the page state  in snp_memcpy()
84  * page(s) that map this memory. It assumes that eventually the memory is
89  * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
104 	 * one page at time.  in __sme_early_enc_dec()
111 		 * the memory. Use a write-protected mapping for the source.  in __sme_early_enc_dec()
127 		 * Use a temporary buffer, of cache-line multiple size, to  in __sme_early_enc_dec()
142 		size -= len;  in __sme_early_enc_dec()
159 	unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;  in __sme_early_map_unmap_mem()
171 		size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;  in __sme_early_map_unmap_mem()
187 	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);  in sme_unmap_bootdata()
209 	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);  in sme_map_bootdata()
217 static unsigned long pg_level_to_pfn(int level, pte_t *kpte, pgprot_t *ret_prot)  in pg_level_to_pfn()  argument
222 	switch (level) {  in pg_level_to_pfn()
236 		WARN_ONCE(1, "Invalid level for kpte\n");  in pg_level_to_pfn()
262 		int psize, pmask, level;  in enc_dec_hypercall()  local
266 		kpte = lookup_address(vaddr, &level);  in enc_dec_hypercall()
272 		pfn = pg_level_to_pfn(level, kpte, NULL);  in enc_dec_hypercall()
276 		psize = page_level_size(level);  in enc_dec_hypercall()
277 		pmask = page_level_mask(level);  in enc_dec_hypercall()
289 	 * To maintain the security guarantees of SEV-SNP guests, make sure  in amd_enc_status_change_prepare()
302 	 * After memory is mapped encrypted in the page table, validate it  in amd_enc_status_change_finish()
303 	 * so that it is consistent with the page table updates.  in amd_enc_status_change_finish()
314 static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)  in __set_clr_pte_enc()  argument
320 	pfn = pg_level_to_pfn(level, kpte, &old_prot);  in __set_clr_pte_enc()
335 	size = page_level_size(level);  in __set_clr_pte_enc()
338 	 * We are going to perform in-place en-/decryption and change the  in __set_clr_pte_enc()
339 	 * physical page attribute from C=1 to C=0 or vice versa. Flush the  in __set_clr_pte_enc()
340 	 * caches to ensure that data gets accessed with the correct C-bit.  in __set_clr_pte_enc()
344 	/* Encrypt/decrypt the contents in-place */  in __set_clr_pte_enc()
351 		 * ON SNP, the page state in the RMP table must happen  in __set_clr_pte_enc()
352 		 * before the page table updates.  in __set_clr_pte_enc()
357 	/* Change the page encryption mask. */  in __set_clr_pte_enc()
362 	 * If page is set encrypted in the page table, then update the RMP table to  in __set_clr_pte_enc()
363 	 * add this page as private.  in __set_clr_pte_enc()
375 	int level, ret;  in early_set_memory_enc_dec()  local
383 		kpte = lookup_address(vaddr, &level);  in early_set_memory_enc_dec()
389 		if (level == PG_LEVEL_4K) {  in early_set_memory_enc_dec()
390 			__set_clr_pte_enc(kpte, level, enc);  in early_set_memory_enc_dec()
395 		psize = page_level_size(level);  in early_set_memory_enc_dec()
396 		pmask = page_level_mask(level);  in early_set_memory_enc_dec()
399 		 * Check whether we can change the large page in one go.  in early_set_memory_enc_dec()
402 		 * than the number of pages in the large page.  in early_set_memory_enc_dec()
405 		    ((vaddr_end - vaddr) >= psize)) {  in early_set_memory_enc_dec()
406 			__set_clr_pte_enc(kpte, level, enc);  in early_set_memory_enc_dec()
412 		 * The virtual address is part of a larger page, create the next  in early_set_memory_enc_dec()
413 		 * level page table mapping (4K or 2M). If it is part of a 2M  in early_set_memory_enc_dec()
414 		 * page then we request a split of the large page into 4K  in early_set_memory_enc_dec()
415 		 * chunks. A 1GB large page is split into 2M pages, resp.  in early_set_memory_enc_dec()
417 		if (level == PG_LEVEL_2M)  in early_set_memory_enc_dec()
472 	 * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the  in sme_early_init()
473 	 * parallel bringup low level code. That raises #VC which cannot be  in sme_early_init()
490 	 * By default, the 32-bit emulation is disabled in order to ensure  in sme_early_init()
504 	npages = (vaddr_end - vaddr) >> PAGE_SHIFT;  in mem_encrypt_free_decrypted_mem()
509 	 * re-encryption on the same condition used for the decryption in  in mem_encrypt_free_decrypted_mem()
510 	 * sme_postprocess_startup(). Higher level abstractions, such as  in mem_encrypt_free_decrypted_mem()
511 	 * CC_ATTR_MEM_ENCRYPT, aren't necessarily equivalent in a Hyper-V VM  in mem_encrypt_free_decrypted_mem()