1/* SPDX-License-Identifier: GPL-2.0 */ 2 3#include <linux/linkage.h> 4#include <asm/segment.h> 5#include <asm/boot.h> 6#include <asm/msr.h> 7#include <asm/processor-flags.h> 8 9/* 10 * This is the 32-bit trampoline that will be copied over to low memory. It 11 * will be called using the ordinary 64-bit calling convention from code 12 * running in 64-bit mode. 13 * 14 * Return address is at the top of the stack (might be above 4G). 15 * The first argument (EDI) contains the address of the temporary PGD level 16 * page table in 32-bit addressable memory which will be programmed into 17 * register CR3. 18 */ 19 20 .section ".rodata", "a", @progbits 21SYM_CODE_START(trampoline_32bit_src) 22 /* 23 * Preserve callee save 64-bit registers on the stack: this is 24 * necessary because the architecture does not guarantee that GPRs will 25 * retain their full 64-bit values across a 32-bit mode switch. 26 */ 27 pushq %r15 28 pushq %r14 29 pushq %r13 30 pushq %r12 31 pushq %rbp 32 pushq %rbx 33 34 /* Preserve top half of RSP in a legacy mode GPR to avoid truncation */ 35 movq %rsp, %rbx 36 shrq $32, %rbx 37 38 /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */ 39 pushq $__KERNEL32_CS 40 leaq 0f(%rip), %rax 41 pushq %rax 42 lretq 43 44 /* 45 * The 32-bit code below will do a far jump back to long mode and end 46 * up here after reconfiguring the number of paging levels. First, the 47 * stack pointer needs to be restored to its full 64-bit value before 48 * the callee save register contents can be popped from the stack. 49 */ 50.Lret: 51 shlq $32, %rbx 52 orq %rbx, %rsp 53 54 /* Restore the preserved 64-bit registers */ 55 popq %rbx 56 popq %rbp 57 popq %r12 58 popq %r13 59 popq %r14 60 popq %r15 61 retq 62 63 .code32 640: 65 /* Disable paging */ 66 movl %cr0, %eax 67 btrl $X86_CR0_PG_BIT, %eax 68 movl %eax, %cr0 69 70 /* Point CR3 to the trampoline's new top level page table */ 71 movl %edi, %cr3 72 73 /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */ 74 movl $MSR_EFER, %ecx 75 rdmsr 76 btsl $_EFER_LME, %eax 77 /* Avoid writing EFER if no change was made (for TDX guest) */ 78 jc 1f 79 wrmsr 801: 81 /* Toggle CR4.LA57 */ 82 movl %cr4, %eax 83 btcl $X86_CR4_LA57_BIT, %eax 84 movl %eax, %cr4 85 86 /* Enable paging again. */ 87 movl %cr0, %eax 88 btsl $X86_CR0_PG_BIT, %eax 89 movl %eax, %cr0 90 91 /* 92 * Return to the 64-bit calling code using LJMP rather than LRET, to 93 * avoid the need for a 32-bit addressable stack. The destination 94 * address will be adjusted after the template code is copied into a 95 * 32-bit addressable buffer. 96 */ 97.Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampoline_32bit_src) 98SYM_CODE_END(trampoline_32bit_src) 99 100/* 101 * This symbol is placed right after trampoline_32bit_src() so its address can 102 * be used to infer the size of the trampoline code. 103 */ 104SYM_DATA(trampoline_ljmp_imm_offset, .word .Ljmp + 1 - trampoline_32bit_src) 105 106 /* 107 * The trampoline code has a size limit. 108 * Make sure we fail to compile if the trampoline code grows 109 * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes. 110 */ 111 .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE 112