x86/kernel/head_64.S

1 /* SPDX-License-Identifier: GPL-2.0 */
3  *  linux/arch/x86/kernel/head_64.S -- start in 32bit and switch to 64bit
5  *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6  *  Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7  *  Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
21 #include <asm/processor-flags.h>
25 #include <asm/nospec-branch.h>
32  * because we need identity-mapped pages.
35 #define pud_index(x)	(((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
69 	leaq	(__end_init_task - PTREGS_SIZE)(%rip), %rsp
74 	movl	$MSR_GS_BASE, %ecx
94 	 * be done now, since this also includes setup of the SEV-SNP CPUID table,
116 	addq	$(early_top_pgt - __START_KERNEL_map), %rax
125 	 * For SEV guests: Verify that the C-bit is correct. A malicious
126 	 * hypervisor could lie about the C-bit position to perform a ROP
162 	 * SEV-ES guests. In those guests the call to verify_cpu() would cause
166 	 * All non SEV-ES systems, especially Intel systems, need to execute
187 	addq	$(init_top_pgt - __START_KERNEL_map), %rax
199 	andl	$X86_CR4_MCE, %ecx
201 	movl	$0, %ecx
205 	orl	$(X86_CR4_PAE | X86_CR4_PSE | X86_CR4_PGE), %ecx
209 	orl	$X86_CR4_LA57, %ecx
214 	/* Setup early boot stage 4-/5-level pagetables. */
218 	 * Switch to new page-table
252 	 * Bit 0-23	CPU# if STARTUP_xx flags are not set
254 	movl	smpboot_control(%rip), %ecx
255 	testl	$STARTUP_READ_APICID, %ecx
259 	 * in bit 0-23. This is also the boot CPU case (CPU number 0).
261 	andl	$(~STARTUP_PARALLEL_MASK), %ecx
266 	mov	$MSR_IA32_APICBASE, %ecx
287 	/* Read the APIC ID from the fix-mapped MMIO space. */
295 	mov	$APIC_X2APIC_ID_MSR, %ecx
306 	inc	%ecx
308 	cmpl	$NR_CPUS, %ecx
310 	cmpl	nr_cpu_ids(%rip), %ecx
323 	/* Get the per cpu offset for the given CPU# which is in ECX */
326 	xorl	%edx, %edx /* zero-extended to clear all of RDX */
330 	 * Setup a boot time stack - Any secondary CPU will have lost its stack
331 	 * by now because the cr3-switch above unmaps the real-mode stack.
333 	 * RDX contains the per-cpu offset
355 	movw	$(GDT_SIZE-1), (%rsp)
382 	movl	$MSR_GS_BASE,%ecx
399 	movl	$MSR_EFER, %ecx
433 	 * Since we are running on identity-mapped space we have to jump
547 	.fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
562 	/* The vector number is currently in the pt_regs->di slot. */
563 	pushq %rsi				/* pt_regs->si */
565 	movq %rdi, 8(%rsp)			/* pt_regs->di = RDI */
566 	pushq %rdx				/* pt_regs->dx */
567 	pushq %rcx				/* pt_regs->cx */
568 	pushq %rax				/* pt_regs->ax */
569 	pushq %r8				/* pt_regs->r8 */
570 	pushq %r9				/* pt_regs->r9 */
571 	pushq %r10				/* pt_regs->r10 */
572 	pushq %r11				/* pt_regs->r11 */
573 	pushq %rbx				/* pt_regs->bx */
574 	pushq %rbp				/* pt_regs->bp */
575 	pushq %r12				/* pt_regs->r12 */
576 	pushq %r13				/* pt_regs->r13 */
577 	pushq %r14				/* pt_regs->r14 */
578 	pushq %r15				/* pt_regs->r15 */
592  * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
617 	/* Pure iret required here - don't use INTERRUPT_RETURN */
637 /* This ensures they are 8k-aligned: */
672 	.quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
674 	.quad   level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
676 	/* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
677 	.quad   level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
682 	.quad	level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
706 	.quad	level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
712 	/* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
713 	.quad	level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
714 	.quad	level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
735 	.fill	(512 - 4 - FIXMAP_PMD_NUM),8,0
738 	.quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
764 #include "../../x86/xen/xen-head.S"