1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  linux/arch/x86/kernel/head_64.S -- start in 32bit and switch to 64bit
4 *
5 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 *  Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *  Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
8 *  Copyright (C) 2001,2002 Andi Kleen <ak@suse.de>
9 *  Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
10 */
11
12#include <linux/export.h>
13#include <linux/linkage.h>
14#include <linux/threads.h>
15#include <linux/init.h>
16#include <linux/pgtable.h>
17#include <asm/segment.h>
18#include <asm/page.h>
19#include <asm/msr.h>
20#include <asm/cache.h>
21#include <asm/processor-flags.h>
22#include <asm/percpu.h>
23#include <asm/nops.h>
24#include "../entry/calling.h"
25#include <asm/nospec-branch.h>
26#include <asm/apicdef.h>
27#include <asm/fixmap.h>
28#include <asm/smp.h>
29#include <asm/thread_info.h>
30
31/*
32 * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
33 * because we need identity-mapped pages.
34 */
35
36	__HEAD
37	.code64
38SYM_CODE_START_NOALIGN(startup_64)
39	UNWIND_HINT_END_OF_STACK
40	/*
41	 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
42	 * and someone has loaded an identity mapped page table
43	 * for us.  These identity mapped page tables map all of the
44	 * kernel pages and possibly all of memory.
45	 *
46	 * %RSI holds the physical address of the boot_params structure
47	 * provided by the bootloader. Preserve it in %R15 so C function calls
48	 * will not clobber it.
49	 *
50	 * We come here either directly from a 64bit bootloader, or from
51	 * arch/x86/boot/compressed/head_64.S.
52	 *
53	 * We only come here initially at boot nothing else comes here.
54	 *
55	 * Since we may be loaded at an address different from what we were
56	 * compiled to run at we first fixup the physical addresses in our page
57	 * tables and then reload them.
58	 */
59	mov	%rsi, %r15
60
61	/* Set up the stack for verify_cpu() */
62	leaq	__top_init_kernel_stack(%rip), %rsp
63
64	/*
65	 * Set up GSBASE.
66	 * Note that on SMP the boot CPU uses the init data section until
67	 * the per-CPU areas are set up.
68	 */
69	movl	$MSR_GS_BASE, %ecx
70	xorl	%eax, %eax
71	xorl	%edx, %edx
72	wrmsr
73
74	call	startup_64_setup_gdt_idt
75
76	/* Now switch to __KERNEL_CS so IRET works reliably */
77	pushq	$__KERNEL_CS
78	leaq	.Lon_kernel_cs(%rip), %rax
79	pushq	%rax
80	lretq
81
82.Lon_kernel_cs:
83	ANNOTATE_NOENDBR
84	UNWIND_HINT_END_OF_STACK
85
86#ifdef CONFIG_AMD_MEM_ENCRYPT
87	/*
88	 * Activate SEV/SME memory encryption if supported/enabled. This needs to
89	 * be done now, since this also includes setup of the SEV-SNP CPUID table,
90	 * which needs to be done before any CPUID instructions are executed in
91	 * subsequent code. Pass the boot_params pointer as the first argument.
92	 */
93	movq	%r15, %rdi
94	call	sme_enable
95#endif
96
97	/* Sanitize CPU configuration */
98	call verify_cpu
99
100	/*
101	 * Derive the kernel's physical-to-virtual offset from the physical and
102	 * virtual addresses of common_startup_64().
103	 */
104	leaq	common_startup_64(%rip), %rdi
105	subq	.Lcommon_startup_64(%rip), %rdi
106
107	/*
108	 * Perform pagetable fixups. Additionally, if SME is active, encrypt
109	 * the kernel and retrieve the modifier (SME encryption mask if SME
110	 * is active) to be added to the initial pgdir entry that will be
111	 * programmed into CR3.
112	 */
113	movq	%r15, %rsi
114	call	__startup_64
115
116	/* Form the CR3 value being sure to include the CR3 modifier */
117	leaq	early_top_pgt(%rip), %rcx
118	addq	%rcx, %rax
119
120#ifdef CONFIG_AMD_MEM_ENCRYPT
121	mov	%rax, %rdi
122
123	/*
124	 * For SEV guests: Verify that the C-bit is correct. A malicious
125	 * hypervisor could lie about the C-bit position to perform a ROP
126	 * attack on the guest by writing to the unencrypted stack and wait for
127	 * the next RET instruction.
128	 */
129	call	sev_verify_cbit
130#endif
131
132	/*
133	 * Switch to early_top_pgt which still has the identity mappings
134	 * present.
135	 */
136	movq	%rax, %cr3
137
138	/* Branch to the common startup code at its kernel virtual address */
139	ANNOTATE_RETPOLINE_SAFE
140	jmp	*.Lcommon_startup_64(%rip)
141SYM_CODE_END(startup_64)
142
143	__INITRODATA
144SYM_DATA_LOCAL(.Lcommon_startup_64, .quad common_startup_64)
145
146	.text
147SYM_CODE_START(secondary_startup_64)
148	UNWIND_HINT_END_OF_STACK
149	ANNOTATE_NOENDBR
150	/*
151	 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
152	 * and someone has loaded a mapped page table.
153	 *
154	 * We come here either from startup_64 (using physical addresses)
155	 * or from trampoline.S (using virtual addresses).
156	 *
157	 * Using virtual addresses from trampoline.S removes the need
158	 * to have any identity mapped pages in the kernel page table
159	 * after the boot processor executes this code.
160	 */
161
162	/* Sanitize CPU configuration */
163	call verify_cpu
164
165	/*
166	 * The secondary_startup_64_no_verify entry point is only used by
167	 * SEV-ES guests. In those guests the call to verify_cpu() would cause
168	 * #VC exceptions which can not be handled at this stage of secondary
169	 * CPU bringup.
170	 *
171	 * All non SEV-ES systems, especially Intel systems, need to execute
172	 * verify_cpu() above to make sure NX is enabled.
173	 */
174SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
175	UNWIND_HINT_END_OF_STACK
176	ANNOTATE_NOENDBR
177
178	/* Clear %R15 which holds the boot_params pointer on the boot CPU */
179	xorl	%r15d, %r15d
180
181	/* Derive the runtime physical address of init_top_pgt[] */
182	movq	phys_base(%rip), %rax
183	addq	$(init_top_pgt - __START_KERNEL_map), %rax
184
185	/*
186	 * Retrieve the modifier (SME encryption mask if SME is active) to be
187	 * added to the initial pgdir entry that will be programmed into CR3.
188	 */
189#ifdef CONFIG_AMD_MEM_ENCRYPT
190	addq	sme_me_mask(%rip), %rax
191#endif
192	/*
193	 * Switch to the init_top_pgt here, away from the trampoline_pgd and
194	 * unmap the identity mapped ranges.
195	 */
196	movq	%rax, %cr3
197
198SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
199	UNWIND_HINT_END_OF_STACK
200	ANNOTATE_NOENDBR
201
202	/*
203	 * Create a mask of CR4 bits to preserve. Omit PGE in order to flush
204	 * global 1:1 translations from the TLBs.
205	 *
206	 * From the SDM:
207	 * "If CR4.PGE is changing from 0 to 1, there were no global TLB
208	 *  entries before the execution; if CR4.PGE is changing from 1 to 0,
209	 *  there will be no global TLB entries after the execution."
210	 */
211	movl	$(X86_CR4_PAE | X86_CR4_LA57), %edx
212#ifdef CONFIG_X86_MCE
213	/*
214	 * Preserve CR4.MCE if the kernel will enable #MC support.
215	 * Clearing MCE may fault in some environments (that also force #MC
216	 * support). Any machine check that occurs before #MC support is fully
217	 * configured will crash the system regardless of the CR4.MCE value set
218	 * here.
219	 */
220	orl	$X86_CR4_MCE, %edx
221#endif
222	movq	%cr4, %rcx
223	andl	%edx, %ecx
224
225	/* Even if ignored in long mode, set PSE uniformly on all logical CPUs. */
226	btsl	$X86_CR4_PSE_BIT, %ecx
227	movq	%rcx, %cr4
228
229	/*
230	 * Set CR4.PGE to re-enable global translations.
231	 */
232	btsl	$X86_CR4_PGE_BIT, %ecx
233	movq	%rcx, %cr4
234
235#ifdef CONFIG_SMP
236	/*
237	 * For parallel boot, the APIC ID is read from the APIC, and then
238	 * used to look up the CPU number.  For booting a single CPU, the
239	 * CPU number is encoded in smpboot_control.
240	 *
241	 * Bit 31	STARTUP_READ_APICID (Read APICID from APIC)
242	 * Bit 0-23	CPU# if STARTUP_xx flags are not set
243	 */
244	movl	smpboot_control(%rip), %ecx
245	testl	$STARTUP_READ_APICID, %ecx
246	jnz	.Lread_apicid
247	/*
248	 * No control bit set, single CPU bringup. CPU number is provided
249	 * in bit 0-23. This is also the boot CPU case (CPU number 0).
250	 */
251	andl	$(~STARTUP_PARALLEL_MASK), %ecx
252	jmp	.Lsetup_cpu
253
254.Lread_apicid:
255	/* Check whether X2APIC mode is already enabled */
256	mov	$MSR_IA32_APICBASE, %ecx
257	rdmsr
258	testl	$X2APIC_ENABLE, %eax
259	jnz	.Lread_apicid_msr
260
261#ifdef CONFIG_X86_X2APIC
262	/*
263	 * If system is in X2APIC mode then MMIO base might not be
264	 * mapped causing the MMIO read below to fault. Faults can't
265	 * be handled at that point.
266	 */
267	cmpl	$0, x2apic_mode(%rip)
268	jz	.Lread_apicid_mmio
269
270	/* Force the AP into X2APIC mode. */
271	orl	$X2APIC_ENABLE, %eax
272	wrmsr
273	jmp	.Lread_apicid_msr
274#endif
275
276.Lread_apicid_mmio:
277	/* Read the APIC ID from the fix-mapped MMIO space. */
278	movq	apic_mmio_base(%rip), %rcx
279	addq	$APIC_ID, %rcx
280	movl	(%rcx), %eax
281	shr	$24, %eax
282	jmp	.Llookup_AP
283
284.Lread_apicid_msr:
285	mov	$APIC_X2APIC_ID_MSR, %ecx
286	rdmsr
287
288.Llookup_AP:
289	/* EAX contains the APIC ID of the current CPU */
290	xorl	%ecx, %ecx
291	leaq	cpuid_to_apicid(%rip), %rbx
292
293.Lfind_cpunr:
294	cmpl	(%rbx,%rcx,4), %eax
295	jz	.Lsetup_cpu
296	inc	%ecx
297#ifdef CONFIG_FORCE_NR_CPUS
298	cmpl	$NR_CPUS, %ecx
299#else
300	cmpl	nr_cpu_ids(%rip), %ecx
301#endif
302	jb	.Lfind_cpunr
303
304	/*  APIC ID not found in the table. Drop the trampoline lock and bail. */
305	movq	trampoline_lock(%rip), %rax
306	movl	$0, (%rax)
307
3081:	cli
309	hlt
310	jmp	1b
311
312.Lsetup_cpu:
313	/* Get the per cpu offset for the given CPU# which is in ECX */
314	movq	__per_cpu_offset(,%rcx,8), %rdx
315#else
316	xorl	%edx, %edx /* zero-extended to clear all of RDX */
317#endif /* CONFIG_SMP */
318
319	/*
320	 * Setup a boot time stack - Any secondary CPU will have lost its stack
321	 * by now because the cr3-switch above unmaps the real-mode stack.
322	 *
323	 * RDX contains the per-cpu offset
324	 */
325	movq	current_task(%rdx), %rax
326	movq	TASK_threadsp(%rax), %rsp
327
328	/*
329	 * Now that this CPU is running on its own stack, drop the realmode
330	 * protection. For the boot CPU the pointer is NULL!
331	 */
332	movq	trampoline_lock(%rip), %rax
333	testq	%rax, %rax
334	jz	.Lsetup_gdt
335	movl	$0, (%rax)
336
337.Lsetup_gdt:
338	/*
339	 * We must switch to a new descriptor in kernel space for the GDT
340	 * because soon the kernel won't have access anymore to the userspace
341	 * addresses where we're currently running on. We have to do that here
342	 * because in 32bit we couldn't load a 64bit linear address.
343	 */
344	subq	$16, %rsp
345	movw	$(GDT_SIZE-1), (%rsp)
346	leaq	gdt_page(%rdx), %rax
347	movq	%rax, 2(%rsp)
348	lgdt	(%rsp)
349	addq	$16, %rsp
350
351	/* set up data segments */
352	xorl %eax,%eax
353	movl %eax,%ds
354	movl %eax,%ss
355	movl %eax,%es
356
357	/*
358	 * We don't really need to load %fs or %gs, but load them anyway
359	 * to kill any stale realmode selectors.  This allows execution
360	 * under VT hardware.
361	 */
362	movl %eax,%fs
363	movl %eax,%gs
364
365	/*
366	 * Set up GSBASE.
367	 * Note that, on SMP, the boot cpu uses init data section until
368	 * the per cpu areas are set up.
369	 */
370	movl	$MSR_GS_BASE,%ecx
371	movl	%edx, %eax
372	shrq	$32, %rdx
373	wrmsr
374
375	/* Setup and Load IDT */
376	call	early_setup_idt
377
378	/* Check if nx is implemented */
379	movl	$0x80000001, %eax
380	cpuid
381	movl	%edx,%edi
382
383	/* Setup EFER (Extended Feature Enable Register) */
384	movl	$MSR_EFER, %ecx
385	rdmsr
386	/*
387	 * Preserve current value of EFER for comparison and to skip
388	 * EFER writes if no change was made (for TDX guest)
389	 */
390	movl    %eax, %edx
391	btsl	$_EFER_SCE, %eax	/* Enable System Call */
392	btl	$20,%edi		/* No Execute supported? */
393	jnc     1f
394	btsl	$_EFER_NX, %eax
395	btsq	$_PAGE_BIT_NX,early_pmd_flags(%rip)
396
397	/* Avoid writing EFER if no change was made (for TDX guest) */
3981:	cmpl	%edx, %eax
399	je	1f
400	xor	%edx, %edx
401	wrmsr				/* Make changes effective */
4021:
403	/* Setup cr0 */
404	movl	$CR0_STATE, %eax
405	/* Make changes effective */
406	movq	%rax, %cr0
407
408	/* zero EFLAGS after setting rsp */
409	pushq $0
410	popfq
411
412	/* Pass the boot_params pointer as first argument */
413	movq	%r15, %rdi
414
415.Ljump_to_C_code:
416	xorl	%ebp, %ebp	# clear frame pointer
417	ANNOTATE_RETPOLINE_SAFE
418	callq	*initial_code(%rip)
419	ud2
420SYM_CODE_END(secondary_startup_64)
421
422#include "verify_cpu.S"
423#include "sev_verify_cbit.S"
424
425#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
426/*
427 * Entry point for soft restart of a CPU. Invoked from xxx_play_dead() for
428 * restarting the boot CPU or for restarting SEV guest CPUs after CPU hot
429 * unplug. Everything is set up already except the stack.
430 */
431SYM_CODE_START(soft_restart_cpu)
432	ANNOTATE_NOENDBR
433	UNWIND_HINT_END_OF_STACK
434
435	/* Find the idle task stack */
436	movq	PER_CPU_VAR(current_task), %rcx
437	movq	TASK_threadsp(%rcx), %rsp
438
439	jmp	.Ljump_to_C_code
440SYM_CODE_END(soft_restart_cpu)
441#endif
442
443#ifdef CONFIG_AMD_MEM_ENCRYPT
444/*
445 * VC Exception handler used during early boot when running on kernel
446 * addresses, but before the switch to the idt_table can be made.
447 * The early_idt_handler_array can't be used here because it calls into a lot
448 * of __init code and this handler is also used during CPU offlining/onlining.
449 * Therefore this handler ends up in the .text section so that it stays around
450 * when .init.text is freed.
451 */
452SYM_CODE_START_NOALIGN(vc_boot_ghcb)
453	UNWIND_HINT_IRET_REGS offset=8
454	ENDBR
455
456	/* Build pt_regs */
457	PUSH_AND_CLEAR_REGS
458
459	/* Call C handler */
460	movq    %rsp, %rdi
461	movq	ORIG_RAX(%rsp), %rsi
462	movq	initial_vc_handler(%rip), %rax
463	ANNOTATE_RETPOLINE_SAFE
464	call	*%rax
465
466	/* Unwind pt_regs */
467	POP_REGS
468
469	/* Remove Error Code */
470	addq    $8, %rsp
471
472	iretq
473SYM_CODE_END(vc_boot_ghcb)
474#endif
475
476	/* Both SMP bootup and ACPI suspend change these variables */
477	__REFDATA
478	.balign	8
479SYM_DATA(initial_code,	.quad x86_64_start_kernel)
480#ifdef CONFIG_AMD_MEM_ENCRYPT
481SYM_DATA(initial_vc_handler,	.quad handle_vc_boot_ghcb)
482#endif
483
484SYM_DATA(trampoline_lock, .quad 0);
485	__FINITDATA
486
487	__INIT
488SYM_CODE_START(early_idt_handler_array)
489	i = 0
490	.rept NUM_EXCEPTION_VECTORS
491	.if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
492		UNWIND_HINT_IRET_REGS
493		ENDBR
494		pushq $0	# Dummy error code, to make stack frame uniform
495	.else
496		UNWIND_HINT_IRET_REGS offset=8
497		ENDBR
498	.endif
499	pushq $i		# 72(%rsp) Vector number
500	jmp early_idt_handler_common
501	UNWIND_HINT_IRET_REGS
502	i = i + 1
503	.fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
504	.endr
505SYM_CODE_END(early_idt_handler_array)
506	ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]
507
508SYM_CODE_START_LOCAL(early_idt_handler_common)
509	UNWIND_HINT_IRET_REGS offset=16
510	/*
511	 * The stack is the hardware frame, an error code or zero, and the
512	 * vector number.
513	 */
514	cld
515
516	incl early_recursion_flag(%rip)
517
518	/* The vector number is currently in the pt_regs->di slot. */
519	pushq %rsi				/* pt_regs->si */
520	movq 8(%rsp), %rsi			/* RSI = vector number */
521	movq %rdi, 8(%rsp)			/* pt_regs->di = RDI */
522	pushq %rdx				/* pt_regs->dx */
523	pushq %rcx				/* pt_regs->cx */
524	pushq %rax				/* pt_regs->ax */
525	pushq %r8				/* pt_regs->r8 */
526	pushq %r9				/* pt_regs->r9 */
527	pushq %r10				/* pt_regs->r10 */
528	pushq %r11				/* pt_regs->r11 */
529	pushq %rbx				/* pt_regs->bx */
530	pushq %rbp				/* pt_regs->bp */
531	pushq %r12				/* pt_regs->r12 */
532	pushq %r13				/* pt_regs->r13 */
533	pushq %r14				/* pt_regs->r14 */
534	pushq %r15				/* pt_regs->r15 */
535	UNWIND_HINT_REGS
536
537	movq %rsp,%rdi		/* RDI = pt_regs; RSI is already trapnr */
538	call do_early_exception
539
540	decl early_recursion_flag(%rip)
541	jmp restore_regs_and_return_to_kernel
542SYM_CODE_END(early_idt_handler_common)
543
544#ifdef CONFIG_AMD_MEM_ENCRYPT
545/*
546 * VC Exception handler used during very early boot. The
547 * early_idt_handler_array can't be used because it returns via the
548 * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
549 *
550 * XXX it does, fix this.
551 *
552 * This handler will end up in the .init.text section and not be
553 * available to boot secondary CPUs.
554 */
555SYM_CODE_START_NOALIGN(vc_no_ghcb)
556	UNWIND_HINT_IRET_REGS offset=8
557	ENDBR
558
559	/* Build pt_regs */
560	PUSH_AND_CLEAR_REGS
561
562	/* Call C handler */
563	movq    %rsp, %rdi
564	movq	ORIG_RAX(%rsp), %rsi
565	call    do_vc_no_ghcb
566
567	/* Unwind pt_regs */
568	POP_REGS
569
570	/* Remove Error Code */
571	addq    $8, %rsp
572
573	/* Pure iret required here - don't use INTERRUPT_RETURN */
574	iretq
575SYM_CODE_END(vc_no_ghcb)
576#endif
577
578#ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
579/*
580 * Each PGD needs to be 8k long and 8k aligned.  We do not
581 * ever go out to userspace with these, so we do not
582 * strictly *need* the second page, but this allows us to
583 * have a single set_pgd() implementation that does not
584 * need to worry about whether it has 4k or 8k to work
585 * with.
586 *
587 * This ensures PGDs are 8k long:
588 */
589#define PTI_USER_PGD_FILL	512
590/* This ensures they are 8k-aligned: */
591#define SYM_DATA_START_PTI_ALIGNED(name) \
592	SYM_START(name, SYM_L_GLOBAL, .balign 2 * PAGE_SIZE)
593#else
594#define SYM_DATA_START_PTI_ALIGNED(name) \
595	SYM_DATA_START_PAGE_ALIGNED(name)
596#define PTI_USER_PGD_FILL	0
597#endif
598
599	__INITDATA
600	.balign 4
601
602SYM_DATA_START_PTI_ALIGNED(early_top_pgt)
603	.fill	511,8,0
604	.quad	level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
605	.fill	PTI_USER_PGD_FILL,8,0
606SYM_DATA_END(early_top_pgt)
607
608SYM_DATA_START_PAGE_ALIGNED(early_dynamic_pgts)
609	.fill	512*EARLY_DYNAMIC_PAGE_TABLES,8,0
610SYM_DATA_END(early_dynamic_pgts)
611
612SYM_DATA(early_recursion_flag, .long 0)
613
614	.data
615
616#if defined(CONFIG_XEN_PV) || defined(CONFIG_PVH)
617SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
618	.quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
619	.org    init_top_pgt + L4_PAGE_OFFSET*8, 0
620	.quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
621	.org    init_top_pgt + L4_START_KERNEL*8, 0
622	/* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
623	.quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
624	.fill	PTI_USER_PGD_FILL,8,0
625SYM_DATA_END(init_top_pgt)
626
627SYM_DATA_START_PAGE_ALIGNED(level3_ident_pgt)
628	.quad	level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
629	.fill	511, 8, 0
630SYM_DATA_END(level3_ident_pgt)
631SYM_DATA_START_PAGE_ALIGNED(level2_ident_pgt)
632	/*
633	 * Since I easily can, map the first 1G.
634	 * Don't set NX because code runs from these pages.
635	 *
636	 * Note: This sets _PAGE_GLOBAL despite whether
637	 * the CPU supports it or it is enabled.  But,
638	 * the CPU should ignore the bit.
639	 */
640	PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
641SYM_DATA_END(level2_ident_pgt)
642#else
643SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
644	.fill	512,8,0
645	.fill	PTI_USER_PGD_FILL,8,0
646SYM_DATA_END(init_top_pgt)
647#endif
648
649#ifdef CONFIG_X86_5LEVEL
650SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt)
651	.fill	511,8,0
652	.quad	level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
653SYM_DATA_END(level4_kernel_pgt)
654#endif
655
656SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt)
657	.fill	L3_START_KERNEL,8,0
658	/* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
659	.quad	level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
660	.quad	level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
661SYM_DATA_END(level3_kernel_pgt)
662
663SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt)
664	/*
665	 * Kernel high mapping.
666	 *
667	 * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in
668	 * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled,
669	 * 512 MiB otherwise.
670	 *
671	 * (NOTE: after that starts the module area, see MODULES_VADDR.)
672	 *
673	 * This table is eventually used by the kernel during normal runtime.
674	 * Care must be taken to clear out undesired bits later, like _PAGE_RW
675	 * or _PAGE_GLOBAL in some cases.
676	 */
677	PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE)
678SYM_DATA_END(level2_kernel_pgt)
679
680SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt)
681	.fill	(512 - 4 - FIXMAP_PMD_NUM),8,0
682	pgtno = 0
683	.rept (FIXMAP_PMD_NUM)
684	.quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
685		+ _PAGE_TABLE_NOENC;
686	pgtno = pgtno + 1
687	.endr
688	/* 6 MB reserved space + a 2MB hole */
689	.fill	4,8,0
690SYM_DATA_END(level2_fixmap_pgt)
691
692SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt)
693	.rept (FIXMAP_PMD_NUM)
694	.fill	512,8,0
695	.endr
696SYM_DATA_END(level1_fixmap_pgt)
697
698	.data
699	.align 16
700
701SYM_DATA(smpboot_control,		.long 0)
702
703	.align 16
704/* This must match the first entry in level2_kernel_pgt */
705SYM_DATA(phys_base, .quad 0x0)
706EXPORT_SYMBOL(phys_base)
707
708#include "../xen/xen-head.S"
709
710	__PAGE_ALIGNED_BSS
711SYM_DATA_START_PAGE_ALIGNED(empty_zero_page)
712	.skip PAGE_SIZE
713SYM_DATA_END(empty_zero_page)
714EXPORT_SYMBOL(empty_zero_page)
715
716