1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6#include <asm/asm-offsets.h>
7#include <asm/asm.h>
8#include <linux/init.h>
9#include <linux/linkage.h>
10#include <asm/thread_info.h>
11#include <asm/page.h>
12#include <asm/pgtable.h>
13#include <asm/csr.h>
14#include <asm/hwcap.h>
15#include <asm/image.h>
16#include <asm/scs.h>
17#include <asm/xip_fixup.h>
18#include "efi-header.S"
19
20__HEAD
21SYM_CODE_START(_start)
22	/*
23	 * Image header expected by Linux boot-loaders. The image header data
24	 * structure is described in asm/image.h.
25	 * Do not modify it without modifying the structure and all bootloaders
26	 * that expects this header format!!
27	 */
28#ifdef CONFIG_EFI
29	/*
30	 * This instruction decodes to "MZ" ASCII required by UEFI.
31	 */
32	c.li s4,-13
33	j _start_kernel
34#else
35	/* jump to start kernel */
36	j _start_kernel
37	/* reserved */
38	.word 0
39#endif
40	.balign 8
41#ifdef CONFIG_RISCV_M_MODE
42	/* Image load offset (0MB) from start of RAM for M-mode */
43	.dword 0
44#else
45#if __riscv_xlen == 64
46	/* Image load offset(2MB) from start of RAM */
47	.dword 0x200000
48#else
49	/* Image load offset(4MB) from start of RAM */
50	.dword 0x400000
51#endif
52#endif
53	/* Effective size of kernel image */
54	.dword _end - _start
55	.dword __HEAD_FLAGS
56	.word RISCV_HEADER_VERSION
57	.word 0
58	.dword 0
59	.ascii RISCV_IMAGE_MAGIC
60	.balign 4
61	.ascii RISCV_IMAGE_MAGIC2
62#ifdef CONFIG_EFI
63	.word pe_head_start - _start
64pe_head_start:
65
66	__EFI_PE_HEADER
67#else
68	.word 0
69#endif
70
71.align 2
72#ifdef CONFIG_MMU
73	.global relocate_enable_mmu
74relocate_enable_mmu:
75	/* Relocate return address */
76	la a1, kernel_map
77	XIP_FIXUP_OFFSET a1
78	REG_L a1, KERNEL_MAP_VIRT_ADDR(a1)
79	la a2, _start
80	sub a1, a1, a2
81	add ra, ra, a1
82
83	/* Point stvec to virtual address of intruction after satp write */
84	la a2, 1f
85	add a2, a2, a1
86	csrw CSR_TVEC, a2
87
88	/* Compute satp for kernel page tables, but don't load it yet */
89	srl a2, a0, PAGE_SHIFT
90	la a1, satp_mode
91	XIP_FIXUP_OFFSET a1
92	REG_L a1, 0(a1)
93	or a2, a2, a1
94
95	/*
96	 * Load trampoline page directory, which will cause us to trap to
97	 * stvec if VA != PA, or simply fall through if VA == PA.  We need a
98	 * full fence here because setup_vm() just wrote these PTEs and we need
99	 * to ensure the new translations are in use.
100	 */
101	la a0, trampoline_pg_dir
102	XIP_FIXUP_OFFSET a0
103	srl a0, a0, PAGE_SHIFT
104	or a0, a0, a1
105	sfence.vma
106	csrw CSR_SATP, a0
107.align 2
1081:
109	/* Set trap vector to spin forever to help debug */
110	la a0, .Lsecondary_park
111	csrw CSR_TVEC, a0
112
113	/* Reload the global pointer */
114	load_global_pointer
115
116	/*
117	 * Switch to kernel page tables.  A full fence is necessary in order to
118	 * avoid using the trampoline translations, which are only correct for
119	 * the first superpage.  Fetching the fence is guaranteed to work
120	 * because that first superpage is translated the same way.
121	 */
122	csrw CSR_SATP, a2
123	sfence.vma
124
125	ret
126#endif /* CONFIG_MMU */
127#ifdef CONFIG_SMP
128	.global secondary_start_sbi
129secondary_start_sbi:
130	/* Mask all interrupts */
131	csrw CSR_IE, zero
132	csrw CSR_IP, zero
133
134#ifndef CONFIG_RISCV_M_MODE
135	/* Enable time CSR */
136	li t0, 0x2
137	csrw CSR_SCOUNTEREN, t0
138#endif
139
140	/* Load the global pointer */
141	load_global_pointer
142
143	/*
144	 * Disable FPU & VECTOR to detect illegal usage of
145	 * floating point or vector in kernel space
146	 */
147	li t0, SR_FS_VS
148	csrc CSR_STATUS, t0
149
150	/* Set trap vector to spin forever to help debug */
151	la a3, .Lsecondary_park
152	csrw CSR_TVEC, a3
153
154	/* a0 contains the hartid & a1 contains boot data */
155	li a2, SBI_HART_BOOT_TASK_PTR_OFFSET
156	XIP_FIXUP_OFFSET a2
157	add a2, a2, a1
158	REG_L tp, (a2)
159	li a3, SBI_HART_BOOT_STACK_PTR_OFFSET
160	XIP_FIXUP_OFFSET a3
161	add a3, a3, a1
162	REG_L sp, (a3)
163
164.Lsecondary_start_common:
165
166#ifdef CONFIG_MMU
167	/* Enable virtual memory and relocate to virtual address */
168	la a0, swapper_pg_dir
169	XIP_FIXUP_OFFSET a0
170	call relocate_enable_mmu
171#endif
172	call .Lsetup_trap_vector
173	scs_load_current
174	call smp_callin
175#endif /* CONFIG_SMP */
176
177.align 2
178.Lsecondary_park:
179	/*
180	 * Park this hart if we:
181	 *  - have too many harts on CONFIG_RISCV_BOOT_SPINWAIT
182	 *  - receive an early trap, before setup_trap_vector finished
183	 *  - fail in smp_callin(), as a successful one wouldn't return
184	 */
185	wfi
186	j .Lsecondary_park
187
188.align 2
189.Lsetup_trap_vector:
190	/* Set trap vector to exception handler */
191	la a0, handle_exception
192	csrw CSR_TVEC, a0
193
194	/*
195	 * Set sup0 scratch register to 0, indicating to exception vector that
196	 * we are presently executing in kernel.
197	 */
198	csrw CSR_SCRATCH, zero
199	ret
200
201SYM_CODE_END(_start)
202
203SYM_CODE_START(_start_kernel)
204	/* Mask all interrupts */
205	csrw CSR_IE, zero
206	csrw CSR_IP, zero
207
208#ifdef CONFIG_RISCV_M_MODE
209	/* flush the instruction cache */
210	fence.i
211
212	/* Reset all registers except ra, a0, a1 */
213	call reset_regs
214
215	/*
216	 * Setup a PMP to permit access to all of memory.  Some machines may
217	 * not implement PMPs, so we set up a quick trap handler to just skip
218	 * touching the PMPs on any trap.
219	 */
220	la a0, .Lpmp_done
221	csrw CSR_TVEC, a0
222
223	li a0, -1
224	csrw CSR_PMPADDR0, a0
225	li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X)
226	csrw CSR_PMPCFG0, a0
227.align 2
228.Lpmp_done:
229
230	/*
231	 * The hartid in a0 is expected later on, and we have no firmware
232	 * to hand it to us.
233	 */
234	csrr a0, CSR_MHARTID
235#else
236	/* Enable time CSR */
237	li t0, 0x2
238	csrw CSR_SCOUNTEREN, t0
239#endif /* CONFIG_RISCV_M_MODE */
240
241	/* Load the global pointer */
242	load_global_pointer
243
244	/*
245	 * Disable FPU & VECTOR to detect illegal usage of
246	 * floating point or vector in kernel space
247	 */
248	li t0, SR_FS_VS
249	csrc CSR_STATUS, t0
250
251#ifdef CONFIG_RISCV_BOOT_SPINWAIT
252	li t0, CONFIG_NR_CPUS
253	blt a0, t0, .Lgood_cores
254	tail .Lsecondary_park
255.Lgood_cores:
256
257	/* The lottery system is only required for spinwait booting method */
258#ifndef CONFIG_XIP_KERNEL
259	/* Pick one hart to run the main boot sequence */
260	la a3, hart_lottery
261	li a2, 1
262	amoadd.w a3, a2, (a3)
263	bnez a3, .Lsecondary_start
264
265#else
266	/* hart_lottery in flash contains a magic number */
267	la a3, hart_lottery
268	mv a2, a3
269	XIP_FIXUP_OFFSET a2
270	XIP_FIXUP_FLASH_OFFSET a3
271	lw t1, (a3)
272	amoswap.w t0, t1, (a2)
273	/* first time here if hart_lottery in RAM is not set */
274	beq t0, t1, .Lsecondary_start
275
276#endif /* CONFIG_XIP */
277#endif /* CONFIG_RISCV_BOOT_SPINWAIT */
278
279#ifdef CONFIG_XIP_KERNEL
280	la sp, _end + THREAD_SIZE
281	XIP_FIXUP_OFFSET sp
282	mv s0, a0
283	mv s1, a1
284	call __copy_data
285
286	/* Restore a0 & a1 copy */
287	mv a0, s0
288	mv a1, s1
289#endif
290
291#ifndef CONFIG_XIP_KERNEL
292	/* Clear BSS for flat non-ELF images */
293	la a3, __bss_start
294	la a4, __bss_stop
295	ble a4, a3, .Lclear_bss_done
296.Lclear_bss:
297	REG_S zero, (a3)
298	add a3, a3, RISCV_SZPTR
299	blt a3, a4, .Lclear_bss
300.Lclear_bss_done:
301#endif
302	la a2, boot_cpu_hartid
303	XIP_FIXUP_OFFSET a2
304	REG_S a0, (a2)
305
306	/* Initialize page tables and relocate to virtual addresses */
307	la tp, init_task
308	la sp, init_thread_union + THREAD_SIZE
309	XIP_FIXUP_OFFSET sp
310	addi sp, sp, -PT_SIZE_ON_STACK
311	scs_load_init_stack
312#ifdef CONFIG_BUILTIN_DTB
313	la a0, __dtb_start
314	XIP_FIXUP_OFFSET a0
315#else
316	mv a0, a1
317#endif /* CONFIG_BUILTIN_DTB */
318	/* Set trap vector to spin forever to help debug */
319	la a3, .Lsecondary_park
320	csrw CSR_TVEC, a3
321	call setup_vm
322#ifdef CONFIG_MMU
323	la a0, early_pg_dir
324	XIP_FIXUP_OFFSET a0
325	call relocate_enable_mmu
326#endif /* CONFIG_MMU */
327
328	call .Lsetup_trap_vector
329	/* Restore C environment */
330	la tp, init_task
331	la sp, init_thread_union + THREAD_SIZE
332	addi sp, sp, -PT_SIZE_ON_STACK
333	scs_load_current
334
335#ifdef CONFIG_KASAN
336	call kasan_early_init
337#endif
338	/* Start the kernel */
339	call soc_early_init
340	tail start_kernel
341
342#ifdef CONFIG_RISCV_BOOT_SPINWAIT
343.Lsecondary_start:
344	/* Set trap vector to spin forever to help debug */
345	la a3, .Lsecondary_park
346	csrw CSR_TVEC, a3
347
348	slli a3, a0, LGREG
349	la a1, __cpu_spinwait_stack_pointer
350	XIP_FIXUP_OFFSET a1
351	la a2, __cpu_spinwait_task_pointer
352	XIP_FIXUP_OFFSET a2
353	add a1, a3, a1
354	add a2, a3, a2
355
356	/*
357	 * This hart didn't win the lottery, so we wait for the winning hart to
358	 * get far enough along the boot process that it should continue.
359	 */
360.Lwait_for_cpu_up:
361	/* FIXME: We should WFI to save some energy here. */
362	REG_L sp, (a1)
363	REG_L tp, (a2)
364	beqz sp, .Lwait_for_cpu_up
365	beqz tp, .Lwait_for_cpu_up
366	fence
367
368	tail .Lsecondary_start_common
369#endif /* CONFIG_RISCV_BOOT_SPINWAIT */
370
371SYM_CODE_END(_start_kernel)
372
373#ifdef CONFIG_RISCV_M_MODE
374SYM_CODE_START_LOCAL(reset_regs)
375	li	sp, 0
376	li	gp, 0
377	li	tp, 0
378	li	t0, 0
379	li	t1, 0
380	li	t2, 0
381	li	s0, 0
382	li	s1, 0
383	li	a2, 0
384	li	a3, 0
385	li	a4, 0
386	li	a5, 0
387	li	a6, 0
388	li	a7, 0
389	li	s2, 0
390	li	s3, 0
391	li	s4, 0
392	li	s5, 0
393	li	s6, 0
394	li	s7, 0
395	li	s8, 0
396	li	s9, 0
397	li	s10, 0
398	li	s11, 0
399	li	t3, 0
400	li	t4, 0
401	li	t5, 0
402	li	t6, 0
403	csrw	CSR_SCRATCH, 0
404
405#ifdef CONFIG_FPU
406	csrr	t0, CSR_MISA
407	andi	t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D)
408	beqz	t0, .Lreset_regs_done_fpu
409
410	li	t1, SR_FS
411	csrs	CSR_STATUS, t1
412	fmv.s.x	f0, zero
413	fmv.s.x	f1, zero
414	fmv.s.x	f2, zero
415	fmv.s.x	f3, zero
416	fmv.s.x	f4, zero
417	fmv.s.x	f5, zero
418	fmv.s.x	f6, zero
419	fmv.s.x	f7, zero
420	fmv.s.x	f8, zero
421	fmv.s.x	f9, zero
422	fmv.s.x	f10, zero
423	fmv.s.x	f11, zero
424	fmv.s.x	f12, zero
425	fmv.s.x	f13, zero
426	fmv.s.x	f14, zero
427	fmv.s.x	f15, zero
428	fmv.s.x	f16, zero
429	fmv.s.x	f17, zero
430	fmv.s.x	f18, zero
431	fmv.s.x	f19, zero
432	fmv.s.x	f20, zero
433	fmv.s.x	f21, zero
434	fmv.s.x	f22, zero
435	fmv.s.x	f23, zero
436	fmv.s.x	f24, zero
437	fmv.s.x	f25, zero
438	fmv.s.x	f26, zero
439	fmv.s.x	f27, zero
440	fmv.s.x	f28, zero
441	fmv.s.x	f29, zero
442	fmv.s.x	f30, zero
443	fmv.s.x	f31, zero
444	csrw	fcsr, 0
445	/* note that the caller must clear SR_FS */
446.Lreset_regs_done_fpu:
447#endif /* CONFIG_FPU */
448
449#ifdef CONFIG_RISCV_ISA_V
450	csrr	t0, CSR_MISA
451	li	t1, COMPAT_HWCAP_ISA_V
452	and	t0, t0, t1
453	beqz	t0, .Lreset_regs_done_vector
454
455	/*
456	 * Clear vector registers and reset vcsr
457	 * VLMAX has a defined value, VLEN is a constant,
458	 * and this form of vsetvli is defined to set vl to VLMAX.
459	 */
460	li	t1, SR_VS
461	csrs	CSR_STATUS, t1
462	csrs	CSR_VCSR, x0
463	vsetvli t1, x0, e8, m8, ta, ma
464	vmv.v.i v0, 0
465	vmv.v.i v8, 0
466	vmv.v.i v16, 0
467	vmv.v.i v24, 0
468	/* note that the caller must clear SR_VS */
469.Lreset_regs_done_vector:
470#endif /* CONFIG_RISCV_ISA_V */
471	ret
472SYM_CODE_END(reset_regs)
473#endif /* CONFIG_RISCV_M_MODE */
474