1/* SPDX-License-Identifier: GPL-2.0 */
2
3#include <linux/linkage.h>
4#include <asm/segment.h>
5#include <asm/boot.h>
6#include <asm/msr.h>
7#include <asm/processor-flags.h>
8#include "pgtable.h"
9
10/*
11 * This is the 32-bit trampoline that will be copied over to low memory. It
12 * will be called using the ordinary 64-bit calling convention from code
13 * running in 64-bit mode.
14 *
15 * Return address is at the top of the stack (might be above 4G).
16 * The first argument (EDI) contains the address of the temporary PGD level
17 * page table in 32-bit addressable memory which will be programmed into
18 * register CR3.
19 */
20
21	.section ".rodata", "a", @progbits
22SYM_CODE_START(trampoline_32bit_src)
23	/*
24	 * Preserve callee save 64-bit registers on the stack: this is
25	 * necessary because the architecture does not guarantee that GPRs will
26	 * retain their full 64-bit values across a 32-bit mode switch.
27	 */
28	pushq	%r15
29	pushq	%r14
30	pushq	%r13
31	pushq	%r12
32	pushq	%rbp
33	pushq	%rbx
34
35	/* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
36	movq	%rsp, %rbx
37	shrq	$32, %rbx
38
39	/* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
40	pushq	$__KERNEL32_CS
41	leaq	0f(%rip), %rax
42	pushq	%rax
43	lretq
44
45	/*
46	 * The 32-bit code below will do a far jump back to long mode and end
47	 * up here after reconfiguring the number of paging levels. First, the
48	 * stack pointer needs to be restored to its full 64-bit value before
49	 * the callee save register contents can be popped from the stack.
50	 */
51.Lret:
52	shlq	$32, %rbx
53	orq	%rbx, %rsp
54
55	/* Restore the preserved 64-bit registers */
56	popq	%rbx
57	popq	%rbp
58	popq	%r12
59	popq	%r13
60	popq	%r14
61	popq	%r15
62	retq
63
64	.code32
650:
66	/* Disable paging */
67	movl	%cr0, %eax
68	btrl	$X86_CR0_PG_BIT, %eax
69	movl	%eax, %cr0
70
71	/* Point CR3 to the trampoline's new top level page table */
72	movl	%edi, %cr3
73
74	/* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
75	movl	$MSR_EFER, %ecx
76	rdmsr
77	btsl	$_EFER_LME, %eax
78	/* Avoid writing EFER if no change was made (for TDX guest) */
79	jc	1f
80	wrmsr
811:
82	/* Toggle CR4.LA57 */
83	movl	%cr4, %eax
84	btcl	$X86_CR4_LA57_BIT, %eax
85	movl	%eax, %cr4
86
87	/* Enable paging again. */
88	movl	%cr0, %eax
89	btsl	$X86_CR0_PG_BIT, %eax
90	movl	%eax, %cr0
91
92	/*
93	 * Return to the 64-bit calling code using LJMP rather than LRET, to
94	 * avoid the need for a 32-bit addressable stack. The destination
95	 * address will be adjusted after the template code is copied into a
96	 * 32-bit addressable buffer.
97	 */
98.Ljmp:	ljmpl	$__KERNEL_CS, $(.Lret - trampoline_32bit_src)
99SYM_CODE_END(trampoline_32bit_src)
100
101/*
102 * This symbol is placed right after trampoline_32bit_src() so its address can
103 * be used to infer the size of the trampoline code.
104 */
105SYM_DATA(trampoline_ljmp_imm_offset, .word  .Ljmp + 1 - trampoline_32bit_src)
106
107	/*
108         * The trampoline code has a size limit.
109         * Make sure we fail to compile if the trampoline code grows
110         * beyond TRAMPOLINE_32BIT_CODE_SIZE bytes.
111	 */
112	.org	trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE
113