1 // SPDX-License-Identifier: GPL-2.0-or-later
6  * memory contents. The most of the code implementation has been adapted
33 #include <asm/fadump-internal.h>
58 #define RESERVED_RNGS_SZ	16384 /* 16K - 128 entries */
63 reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
71  * fadump_cma_init() - Initialize CMA area from a fadump reserved memory
73  * This function initializes CMA area from fadump reserved memory.
74  * The total size of fadump reserved memory covers for boot memory size
76  * Initialize only the area equivalent to boot memory size for CMA use.
77  * The remaining portion of fadump reserved memory will be not given
78  * to CMA and pages for those will stay reserved. boot memory size is
80  * But for some reason even if it fails we still have the memory reservation
106 		pr_err("Failed to init cma area for firmware-assisted dump,%d\n", rc);  in fadump_cma_init()
108 		 * Though the CMA init has failed we still have memory  in fadump_cma_init()
109 		 * reservation with us. The reserved memory will be  in fadump_cma_init()
117 	 *  If CMA activation fails, keep the pages reserved, instead of  in fadump_cma_init()
126 		"bytes of memory reserved for firmware-assisted dump\n",  in fadump_cma_init()
162  * If fadump is registered, check if the memory provided
163  * falls within boot memory area and reserved memory area.
196  * Returns true, if there are no holes in memory area between d_start to d_end,
209 			/* Memory hole from d_start to start */  in is_fadump_mem_area_contiguous()
226  * Returns true, if there are no holes in boot memory area,
248  * Returns true, if there are no holes in reserved memory area,
265 	pr_debug("Support for firmware-assisted dump (fadump): %s\n",  in fadump_show_config()
278 	pr_debug("    Boot memory size   : %lx\n", fw_dump.boot_memory_size);  in fadump_show_config()
279 	pr_debug("    Boot memory top    : %llx\n", fw_dump.boot_mem_top);  in fadump_show_config()
280 	pr_debug("Boot memory regions cnt: %llx\n", fw_dump.boot_mem_regs_cnt);  in fadump_show_config()
290  * Function to find the largest memory size we need to reserve during early
291  * boot process. This will be the size of the memory that is required for a
294  * This function has been taken from phyp-assisted dump feature implementation.
298  * TODO: Come up with better approach to find out more accurate memory size
313 	 * memory at a predefined offset.  in fadump_calculate_reserve_size()
321 			pr_info("Using 'crashkernel=' parameter for memory reservation.\n");  in fadump_calculate_reserve_size()
326 		 * Adjust if the boot memory size specified is above  in fadump_calculate_reserve_size()
332 			pr_info("Adjusted boot memory size to %luMB\n",  in fadump_calculate_reserve_size()
339 		 * 'fadump_reserve_mem=' is being used to reserve memory  in fadump_calculate_reserve_size()
340 		 * for firmware-assisted dump.  in fadump_calculate_reserve_size()
351 	/* Truncate to memory_limit. We don't want to over reserve the memory.*/  in fadump_calculate_reserve_size()
355 	bootmem_min = fw_dump.ops->fadump_get_bootmem_min();  in fadump_calculate_reserve_size()
360  * Calculate the total memory size required to be reserved for
361  * firmware-assisted dump registration.
370 	 * Account for pagesize alignment of boot memory area destination address.  in get_fadump_area_size()
371 	 * This faciliates in mmap reading of first kernel's memory.  in get_fadump_area_size()
378 	/* Program headers for crash memory regions. */  in get_fadump_area_size()
379 	size += sizeof(struct elf_phdr) * (memblock_num_regions(memory) + 2);  in get_fadump_area_size()
384 	size += (fw_dump.ops->fadump_get_metadata_size ?  in get_fadump_area_size()
385 		 fw_dump.ops->fadump_get_metadata_size() : 0);  in get_fadump_area_size()
399 	pr_debug("Added boot memory range[%d] [%#016lx-%#016lx)\n",  in add_boot_mem_region()
408  * Honour that by splitting a memory range into multiple regions.
428 		msize -= rsize;  in add_boot_mem_regions()
449 		size = reg_end - reg_start;  in fadump_get_boot_mem_regions()
450 		hole_size += (reg_start - last_end);  in fadump_get_boot_mem_regions()
453 			size = (mem_size - cur_size);  in fadump_get_boot_mem_regions()
458 		mem_size -= size;  in fadump_get_boot_mem_regions()
472  * Returns true, if the given range overlaps with reserved memory ranges
473  * starting at idx. Also, updates idx to index of overlapping memory range
474  * with the given memory range.
500  * Locate a suitable memory area to reserve memory for FADump. While at it,
501  * lookup reserved-ranges & avoid overlap with them, as they are used by F/W.
519 		while ((mend > base) && ((mend - base) >= size)) {  in fadump_locate_reserve_mem()
543 		pr_info("Firmware-Assisted Dump is not supported on this hardware\n");  in fadump_reserve_mem()
548 	 * Initialize boot memory size  in fadump_reserve_mem()
563 		bootmem_min = fw_dump.ops->fadump_get_bootmem_min();  in fadump_reserve_mem()
565 			pr_err("Can't enable fadump with boot memory size (0x%lx) less than 0x%llx\n",  in fadump_reserve_mem()
571 			pr_err("Too many holes in boot memory area to enable fadump\n");  in fadump_reserve_mem()
577 	 * Calculate the memory boundary.  in fadump_reserve_mem()
578 	 * If memory_limit is less than actual memory boundary then reserve  in fadump_reserve_mem()
579 	 * the memory for fadump beyond the memory_limit and adjust the  in fadump_reserve_mem()
589 		printk(KERN_INFO "Adjusted memory_limit for firmware-assisted"  in fadump_reserve_mem()
601 		pr_info("Firmware-assisted dump is active.\n");  in fadump_reserve_mem()
612 		 * If last boot has crashed then reserve all the memory  in fadump_reserve_mem()
613 		 * above boot memory size so that we don't touch it until  in fadump_reserve_mem()
614 		 * dump is written to disk by userspace tool. This memory  in fadump_reserve_mem()
624 		 * Reserve memory at an offset closer to bottom of the RAM to  in fadump_reserve_mem()
625 		 * minimize the impact of memory hot-remove operation.  in fadump_reserve_mem()
630 			pr_err("Failed to find memory chunk for reservation!\n");  in fadump_reserve_mem()
639 		if (fw_dump.ops->fadump_setup_metadata &&  in fadump_reserve_mem()
640 		    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))  in fadump_reserve_mem()
644 			pr_err("Failed to reserve memory!\n");  in fadump_reserve_mem()
648 		pr_info("Reserved %lldMB of memory at %#016llx (System RAM: %lldMB)\n",  in fadump_reserve_mem()
699 	unsigned int ncpus = num_online_cpus() - 1;  in crash_fadump()
705 	 * old_cpu == -1 means this is the first CPU which has come here,  in crash_fadump()
708 	 * old_cpu != -1 means some other CPU has already on it's way  in crash_fadump()
712 	old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu);  in crash_fadump()
714 	if (old_cpu != -1) {  in crash_fadump()
719 		 * is in force. If we race with fadump un-registration this  in crash_fadump()
730 	fdh->crashing_cpu = crashing_cpu;  in crash_fadump()
734 		fdh->regs = *regs;  in crash_fadump()
736 		ppc_save_regs(&fdh->regs);  in crash_fadump()
738 	fdh->cpu_mask = *cpu_online_mask;  in crash_fadump()
744 	if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {  in crash_fadump()
746 		while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))  in crash_fadump()
750 	fw_dump.ops->fadump_trigger(fdh, str);  in crash_fadump()
777 	if (phdr->p_type == PT_NOTE) {  in fadump_update_elfcore_header()
778 		phdr->p_paddr	= __pa(fw_dump.cpu_notes_buf_vaddr);  in fadump_update_elfcore_header()
779 		phdr->p_offset	= phdr->p_paddr;  in fadump_update_elfcore_header()
780 		phdr->p_filesz	= fw_dump.cpu_notes_buf_size;  in fadump_update_elfcore_header()
781 		phdr->p_memsz = fw_dump.cpu_notes_buf_size;  in fadump_update_elfcore_header()
805 	free_reserved_area((void *)vaddr, (void *)(vaddr + size), -1, NULL);  in fadump_free_buffer()
818 		return -ENOMEM;  in fadump_setup_cpu_notes_buf()
840 	if (mrange_info->is_static) {  in fadump_free_mem_ranges()
841 		mrange_info->mem_range_cnt = 0;  in fadump_free_mem_ranges()
845 	kfree(mrange_info->mem_ranges);  in fadump_free_mem_ranges()
847 	       (sizeof(struct fadump_mrange_info) - RNG_NAME_SZ));  in fadump_free_mem_ranges()
859 	new_size = mrange_info->mem_ranges_sz + PAGE_SIZE;  in fadump_alloc_mem_ranges()
860 	pr_debug("Allocating %llu bytes of memory for %s memory ranges\n",  in fadump_alloc_mem_ranges()
861 		 new_size, mrange_info->name);  in fadump_alloc_mem_ranges()
863 	new_array = krealloc(mrange_info->mem_ranges, new_size, GFP_KERNEL);  in fadump_alloc_mem_ranges()
865 		pr_err("Insufficient memory for setting up %s memory ranges\n",  in fadump_alloc_mem_ranges()
866 		       mrange_info->name);  in fadump_alloc_mem_ranges()
868 		return -ENOMEM;  in fadump_alloc_mem_ranges()
871 	mrange_info->mem_ranges = new_array;  in fadump_alloc_mem_ranges()
872 	mrange_info->mem_ranges_sz = new_size;  in fadump_alloc_mem_ranges()
873 	mrange_info->max_mem_ranges = (new_size /  in fadump_alloc_mem_ranges()
880 	struct fadump_memory_range *mem_ranges = mrange_info->mem_ranges;  in fadump_add_mem_range()
888 	 * Fold adjacent memory ranges to bring down the memory ranges/  in fadump_add_mem_range()
891 	if (mrange_info->mem_range_cnt) {  in fadump_add_mem_range()
892 		start = mem_ranges[mrange_info->mem_range_cnt - 1].base;  in fadump_add_mem_range()
893 		size  = mem_ranges[mrange_info->mem_range_cnt - 1].size;  in fadump_add_mem_range()
896 		 * Boot memory area needs separate PT_LOAD segment(s) as it  in fadump_add_mem_range()
898 		 * So, fold only if the region is not boot memory area.  in fadump_add_mem_range()
905 		if (mrange_info->mem_range_cnt == mrange_info->max_mem_ranges) {  in fadump_add_mem_range()
908 			if (mrange_info->is_static) {  in fadump_add_mem_range()
909 				pr_err("Reached array size limit for %s memory ranges\n",  in fadump_add_mem_range()
910 				       mrange_info->name);  in fadump_add_mem_range()
911 				return -ENOSPC;  in fadump_add_mem_range()
919 			mem_ranges = mrange_info->mem_ranges;  in fadump_add_mem_range()
923 		mem_ranges[mrange_info->mem_range_cnt].base = start;  in fadump_add_mem_range()
924 		mrange_info->mem_range_cnt++;  in fadump_add_mem_range()
927 	mem_ranges[mrange_info->mem_range_cnt - 1].size = (end - start);  in fadump_add_mem_range()
928 	pr_debug("%s_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n",  in fadump_add_mem_range()
929 		 mrange_info->name, (mrange_info->mem_range_cnt - 1),  in fadump_add_mem_range()
930 		 start, end - 1, (end - start));  in fadump_add_mem_range()
970 	memcpy(elf->e_ident, ELFMAG, SELFMAG);  in fadump_init_elfcore_header()
971 	elf->e_ident[EI_CLASS] = ELF_CLASS;  in fadump_init_elfcore_header()
972 	elf->e_ident[EI_DATA] = ELF_DATA;  in fadump_init_elfcore_header()
973 	elf->e_ident[EI_VERSION] = EV_CURRENT;  in fadump_init_elfcore_header()
974 	elf->e_ident[EI_OSABI] = ELF_OSABI;  in fadump_init_elfcore_header()
975 	memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD);  in fadump_init_elfcore_header()
976 	elf->e_type = ET_CORE;  in fadump_init_elfcore_header()
977 	elf->e_machine = ELF_ARCH;  in fadump_init_elfcore_header()
978 	elf->e_version = EV_CURRENT;  in fadump_init_elfcore_header()
979 	elf->e_entry = 0;  in fadump_init_elfcore_header()
980 	elf->e_phoff = sizeof(struct elfhdr);  in fadump_init_elfcore_header()
981 	elf->e_shoff = 0;  in fadump_init_elfcore_header()
984 		elf->e_flags = 2;  in fadump_init_elfcore_header()
986 		elf->e_flags = 1;  in fadump_init_elfcore_header()
988 		elf->e_flags = 0;  in fadump_init_elfcore_header()
990 	elf->e_ehsize = sizeof(struct elfhdr);  in fadump_init_elfcore_header()
991 	elf->e_phentsize = sizeof(struct elf_phdr);  in fadump_init_elfcore_header()
992 	elf->e_phnum = 0;  in fadump_init_elfcore_header()
993 	elf->e_shentsize = 0;  in fadump_init_elfcore_header()
994 	elf->e_shnum = 0;  in fadump_init_elfcore_header()
995 	elf->e_shstrndx = 0;  in fadump_init_elfcore_header()
1001  * Traverse through memblock structure and setup crash memory ranges. These
1009 	pr_debug("Setup crash memory ranges.\n");  in fadump_setup_crash_memory_ranges()
1013 	 * Boot memory region(s) registered with firmware are moved to  in fadump_setup_crash_memory_ranges()
1015 	 * header(s) for this memory chunk(s) with the correct offset.  in fadump_setup_crash_memory_ranges()
1027 		 * skip the memory chunk that is already added  in fadump_setup_crash_memory_ranges()
1037 		/* add this range excluding the reserved dump area. */  in fadump_setup_crash_memory_ranges()
1047  * If the given physical address falls within the boot memory region then
1048  * return the relocated address that points to the dump region reserved
1049  * for saving initial boot memory contents.
1062 		hole_size += (rstart - rlast);  in fadump_relocate()
1065 			raddr += fw_dump.boot_mem_dest_addr - hole_size;  in fadump_relocate()
1096 	phdr->p_type = PT_NOTE;  in fadump_create_elfcore_headers()
1097 	phdr->p_flags = 0;  in fadump_create_elfcore_headers()
1098 	phdr->p_vaddr = 0;  in fadump_create_elfcore_headers()
1099 	phdr->p_align = 0;  in fadump_create_elfcore_headers()
1101 	phdr->p_offset = 0;  in fadump_create_elfcore_headers()
1102 	phdr->p_paddr = 0;  in fadump_create_elfcore_headers()
1103 	phdr->p_filesz = 0;  in fadump_create_elfcore_headers()
1104 	phdr->p_memsz = 0;  in fadump_create_elfcore_headers()
1106 	(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1111 	phdr->p_type	= PT_NOTE;  in fadump_create_elfcore_headers()
1112 	phdr->p_flags	= 0;  in fadump_create_elfcore_headers()
1113 	phdr->p_vaddr	= 0;  in fadump_create_elfcore_headers()
1114 	phdr->p_align	= 0;  in fadump_create_elfcore_headers()
1116 	phdr->p_paddr	= fadump_relocate(paddr_vmcoreinfo_note());  in fadump_create_elfcore_headers()
1117 	phdr->p_offset	= phdr->p_paddr;  in fadump_create_elfcore_headers()
1118 	phdr->p_memsz	= phdr->p_filesz = VMCOREINFO_NOTE_SIZE;  in fadump_create_elfcore_headers()
1121 	(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1137 		phdr->p_type	= PT_LOAD;  in fadump_create_elfcore_headers()
1138 		phdr->p_flags	= PF_R|PF_W|PF_X;  in fadump_create_elfcore_headers()
1139 		phdr->p_offset	= mbase;  in fadump_create_elfcore_headers()
1143 			 * The entire real memory region will be moved by  in fadump_create_elfcore_headers()
1147 			phdr->p_offset = fw_dump.boot_mem_dest_addr + offset;  in fadump_create_elfcore_headers()
1148 			if (j < (fw_dump.boot_mem_regs_cnt - 1)) {  in fadump_create_elfcore_headers()
1154 		phdr->p_paddr = mbase;  in fadump_create_elfcore_headers()
1155 		phdr->p_vaddr = (unsigned long)__va(mbase);  in fadump_create_elfcore_headers()
1156 		phdr->p_filesz = msize;  in fadump_create_elfcore_headers()
1157 		phdr->p_memsz = msize;  in fadump_create_elfcore_headers()
1158 		phdr->p_align = 0;  in fadump_create_elfcore_headers()
1161 		(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1177 	fdh->magic_number = FADUMP_CRASH_INFO_MAGIC;  in init_fadump_header()
1178 	fdh->elfcorehdr_addr = addr;  in init_fadump_header()
1180 	fdh->crashing_cpu = FADUMP_CPU_UNKNOWN;  in init_fadump_header()
1185 	fdh->cpu_mask = *cpu_possible_mask;  in init_fadump_header()
1197 	 * If no memory is reserved then we can not register for firmware-  in register_fadump()
1201 		return -ENODEV;  in register_fadump()
1217 	pr_debug("Registering for firmware-assisted kernel dump...\n");  in register_fadump()
1218 	return fw_dump.ops->fadump_register(&fw_dump);  in register_fadump()
1228 		pr_debug("Invalidating firmware-assisted dump registration\n");  in fadump_cleanup()
1229 		fw_dump.ops->fadump_invalidate(&fw_dump);  in fadump_cleanup()
1231 		/* Un-register Firmware-assisted dump if it was registered. */  in fadump_cleanup()
1232 		fw_dump.ops->fadump_unregister(&fw_dump);  in fadump_cleanup()
1236 	if (fw_dump.ops->fadump_cleanup)  in fadump_cleanup()
1237 		fw_dump.ops->fadump_cleanup(&fw_dump);  in fadump_cleanup()
1246 	pr_info("freeing reserved memory (0x%llx - 0x%llx)\n",  in fadump_free_reserved_memory()
1260  * Skip memory holes and free memory that was actually reserved.
1287  * Sort the mem ranges in-place and merge adjacent ranges
1288  * to minimize the memory ranges count.
1299 	/* Sort the memory ranges */  in sort_and_merge_mem_ranges()
1300 	mem_ranges = mrange_info->mem_ranges;  in sort_and_merge_mem_ranges()
1301 	for (i = 0; i < mrange_info->mem_range_cnt; i++) {  in sort_and_merge_mem_ranges()
1303 		for (j = (i + 1); j < mrange_info->mem_range_cnt; j++) {  in sort_and_merge_mem_ranges()
1311 	/* Merge adjacent reserved ranges */  in sort_and_merge_mem_ranges()
1313 	for (i = 1; i < mrange_info->mem_range_cnt; i++) {  in sort_and_merge_mem_ranges()
1314 		base = mem_ranges[i-1].base;  in sort_and_merge_mem_ranges()
1315 		size = mem_ranges[i-1].size;  in sort_and_merge_mem_ranges()
1326 	mrange_info->mem_range_cnt = idx + 1;  in sort_and_merge_mem_ranges()
1330  * Scan reserved-ranges to consider them while reserving/releasing
1331  * memory for FADump.
1336 	int len, ret = -1;  in early_init_dt_scan_reserved_ranges()
1339 	/* reserved-ranges already scanned */  in early_init_dt_scan_reserved_ranges()
1343 	prop = of_get_flat_dt_prop(node, "reserved-ranges", &len);  in early_init_dt_scan_reserved_ranges()
1348 	 * Each reserved range is an (address,size) pair, 2 cells each,  in early_init_dt_scan_reserved_ranges()
1361 				pr_warn("some reserved ranges are ignored!\n");  in early_init_dt_scan_reserved_ranges()
1367 	/* Compact reserved ranges */  in early_init_dt_scan_reserved_ranges()
1372  * Release the memory that was reserved during early boot to preserve the
1373  * crash'ed kernel's memory contents except reserved dump area (permanent
1374  * reservation) and reserved ranges used by F/W. The released memory will
1386 	 * If reserved ranges array limit is hit, overwrite the last reserved  in fadump_release_memory()
1387 	 * memory range with reserved dump area to ensure it is excluded from  in fadump_release_memory()
1388 	 * the memory being released (reused for next FADump registration).  in fadump_release_memory()
1392 		reserved_mrange_info.mem_range_cnt--;  in fadump_release_memory()
1398 	/* Get the reserved ranges list in order first. */  in fadump_release_memory()
1401 	/* Exclude reserved ranges and release remaining memory */  in fadump_release_memory()
1435 	 * memory structure for FADump re-registration.  in fadump_invalidate_release_mem()
1437 	if (fw_dump.ops->fadump_setup_metadata &&  in fadump_invalidate_release_mem()
1438 	    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))  in fadump_invalidate_release_mem()
1440 	fw_dump.ops->fadump_init_mem_struct(&fw_dump);  in fadump_invalidate_release_mem()
1447 	int input = -1;  in release_mem_store()
1450 		return -EPERM;  in release_mem_store()
1453 		return -EINVAL;  in release_mem_store()
1458 		 * memory, hence it will not be valid anymore.  in release_mem_store()
1466 		return -EINVAL;  in release_mem_store()
1470 /* Release the reserved memory and disable the FADump */
1506 	int input = -1;  in registered_store()
1509 		return -EPERM;  in registered_store()
1512 		return -EINVAL;  in registered_store()
1522 		/* Un-register Firmware-assisted dump */  in registered_store()
1523 		pr_debug("Un-register firmware-assisted dump\n");  in registered_store()
1524 		fw_dump.ops->fadump_unregister(&fw_dump);  in registered_store()
1528 			/* Un-register Firmware-assisted dump */  in registered_store()
1529 			fw_dump.ops->fadump_unregister(&fw_dump);  in registered_store()
1531 		/* Register Firmware-assisted dump */  in registered_store()
1535 		ret = -EINVAL;  in registered_store()
1550 	fw_dump.ops->fadump_region_show(&fw_dump, m);  in fadump_region_show()
1603 	 *      - fadump_enabled -> fadump/enabled  in fadump_init_files()
1604 	 *      - fadump_registered -> fadump/registered  in fadump_init_files()
1605 	 *      - fadump_release_mem -> fadump/release_mem  in fadump_init_files()
1636  * Prepare for firmware-assisted dump.
1656 		 * and release memory before proceeding for re-registration.  in setup_fadump()
1658 		if (fw_dump.ops->fadump_process(&fw_dump) < 0)  in setup_fadump()
1661 	/* Initialize the kernel dump memory structure and register with f/w */  in setup_fadump()
1663 		fw_dump.ops->fadump_init_mem_struct(&fw_dump);  in setup_fadump()
1698  * preserve crash data. The subsequent memory preserving kernel boot
1705 		 * If last boot has crashed then reserve all the memory  in fadump_reserve_mem()
1706 		 * above boot memory to preserve crash data.  in fadump_reserve_mem()
1711 		pr_debug("FADump-aware kernel..\n");  in fadump_reserve_mem()
1723 		msize  = mend - mstart;  in fadump_reserve_crash_area()
1729 			msize -= (base - mstart);  in fadump_reserve_crash_area()
1733 		pr_info("Reserving %lluMB of memory at %#016llx for preserving crash data",  in fadump_reserve_crash_area()