23 #include <linux/cpu.h>
42 #include <asm/cpu.h>
82 static void ipi_setup(int cpu);
85 static void ipi_teardown(int cpu);
86 static int op_cpu_kill(unsigned int cpu);
88 static inline int op_cpu_kill(unsigned int cpu)  in op_cpu_kill()  argument
96  * Boot a secondary CPU, and assign it the specified idle task.
97  * This also gives us the initial stack to use for this CPU.
99 static int boot_secondary(unsigned int cpu, struct task_struct *idle)  in boot_secondary()  argument
101 	const struct cpu_operations *ops = get_cpu_ops(cpu);  in boot_secondary()
104 		return ops->cpu_boot(cpu);  in boot_secondary()
111 int __cpu_up(unsigned int cpu, struct task_struct *idle)  in __cpu_up()  argument
123 	/* Now bring the CPU into our world */  in __cpu_up()
124 	ret = boot_secondary(cpu, idle);  in __cpu_up()
127 			pr_err("CPU%u: failed to boot: %d\n", cpu, ret);  in __cpu_up()
132 	 * CPU was successfully started, wait for it to come online or  in __cpu_up()
137 	if (cpu_online(cpu))  in __cpu_up()
140 	pr_crit("CPU%u: failed to come online\n", cpu);  in __cpu_up()
148 		pr_err("CPU%u: failed in unknown state : 0x%lx\n",  in __cpu_up()
149 		       cpu, status);  in __cpu_up()
153 		if (!op_cpu_kill(cpu)) {  in __cpu_up()
154 			pr_crit("CPU%u: died during early boot\n", cpu);  in __cpu_up()
157 		pr_crit("CPU%u: may not have shut down cleanly\n", cpu);  in __cpu_up()
160 		pr_crit("CPU%u: is stuck in kernel\n", cpu);  in __cpu_up()
162 			pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);  in __cpu_up()
164 			pr_crit("CPU%u: does not support %luK granule\n",  in __cpu_up()
165 				cpu, PAGE_SIZE / SZ_1K);  in __cpu_up()
170 		panic("CPU%u detected unsupported configuration\n", cpu);  in __cpu_up()
192  * This is the secondary CPU boot entry.  We're using this CPUs
200 	unsigned int cpu = smp_processor_id();  in secondary_start_kernel()  local
218 	rcutree_report_cpu_starting(cpu);  in secondary_start_kernel()
223 	 * this CPU ticks all of those. If it doesn't, the CPU will  in secondary_start_kernel()
228 	ops = get_cpu_ops(cpu);  in secondary_start_kernel()
233 	 * Log the CPU info before it is marked online and might get read.  in secondary_start_kernel()
236 	store_cpu_topology(cpu);  in secondary_start_kernel()
241 	notify_cpu_starting(cpu);  in secondary_start_kernel()
243 	ipi_setup(cpu);  in secondary_start_kernel()
245 	numa_add_cpu(cpu);  in secondary_start_kernel()
248 	 * OK, now it's safe to let the boot CPU continue.  Wait for  in secondary_start_kernel()
249 	 * the CPU migration code to notice that the CPU is online  in secondary_start_kernel()
252 	pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",  in secondary_start_kernel()
253 					 cpu, (unsigned long)mpidr,  in secondary_start_kernel()
256 	set_cpu_online(cpu, true);  in secondary_start_kernel()
275 static int op_cpu_disable(unsigned int cpu)  in op_cpu_disable()  argument
277 	const struct cpu_operations *ops = get_cpu_ops(cpu);  in op_cpu_disable()
291 		return ops->cpu_disable(cpu);  in op_cpu_disable()
301 	unsigned int cpu = smp_processor_id();  in __cpu_disable()  local
304 	ret = op_cpu_disable(cpu);  in __cpu_disable()
308 	remove_cpu_topology(cpu);  in __cpu_disable()
309 	numa_remove_cpu(cpu);  in __cpu_disable()
312 	 * Take this CPU offline.  Once we clear this, we can't return,  in __cpu_disable()
313 	 * and we must not schedule until we're ready to give up the cpu.  in __cpu_disable()
315 	set_cpu_online(cpu, false);  in __cpu_disable()
316 	ipi_teardown(cpu);  in __cpu_disable()
319 	 * OK - migrate IRQs away from this CPU  in __cpu_disable()
326 static int op_cpu_kill(unsigned int cpu)  in op_cpu_kill()  argument
328 	const struct cpu_operations *ops = get_cpu_ops(cpu);  in op_cpu_kill()
331 	 * If we have no means of synchronising with the dying CPU, then assume  in op_cpu_kill()
338 	return ops->cpu_kill(cpu);  in op_cpu_kill()
342  * Called on the thread which is asking for a CPU to be shutdown after the
345 void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)  in arch_cpuhp_cleanup_dead_cpu()  argument
349 	pr_debug("CPU%u: shutdown\n", cpu);  in arch_cpuhp_cleanup_dead_cpu()
352 	 * Now that the dying CPU is beyond the point of no return w.r.t.  in arch_cpuhp_cleanup_dead_cpu()
357 	err = op_cpu_kill(cpu);  in arch_cpuhp_cleanup_dead_cpu()
359 		pr_warn("CPU%d may not have shut down cleanly: %d\n", cpu, err);  in arch_cpuhp_cleanup_dead_cpu()
363  * Called from the idle thread for the CPU which has been shutdown.
368 	unsigned int cpu = smp_processor_id();  in cpu_die()  local
369 	const struct cpu_operations *ops = get_cpu_ops(cpu);  in cpu_die()
375 	/* Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose of */  in cpu_die()
379 	 * Actually shutdown the CPU. This must never fail. The specific hotplug  in cpu_die()
381 	 * no dirty lines are lost in the process of shutting down the CPU.  in cpu_die()
383 	ops->cpu_die(cpu);  in cpu_die()
389 static void __cpu_try_die(int cpu)  in __cpu_try_die()  argument
392 	const struct cpu_operations *ops = get_cpu_ops(cpu);  in __cpu_try_die()
395 		ops->cpu_die(cpu);  in __cpu_try_die()
400  * Kill the calling secondary CPU, early in bringup before it is turned
405 	int cpu = smp_processor_id();  in cpu_die_early()  local
407 	pr_crit("CPU%d: will not boot\n", cpu);  in cpu_die_early()
409 	/* Mark this CPU absent */  in cpu_die_early()
410 	set_cpu_present(cpu, 0);  in cpu_die_early()
415 		__cpu_try_die(cpu);  in cpu_die_early()
426 		pr_info("CPU: All CPU(s) started at EL2\n");  in hyp_mode_check()
429 			   "CPU: CPUs started in inconsistent modes");  in hyp_mode_check()
431 		pr_info("CPU: All CPU(s) started at EL1\n");  in hyp_mode_check()
450 	 * The runtime per-cpu areas have been allocated by  in smp_prepare_boot_cpu()
451 	 * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be  in smp_prepare_boot_cpu()
452 	 * freed shortly, so we must move over to the runtime per-cpu area.  in smp_prepare_boot_cpu()
471  * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
474 static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)  in is_mpidr_duplicate()  argument
478 	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)  in is_mpidr_duplicate()
485  * Initialize cpu operations for a logical cpu and
488 static int __init smp_cpu_setup(int cpu)  in smp_cpu_setup()  argument
492 	if (init_cpu_ops(cpu))  in smp_cpu_setup()
495 	ops = get_cpu_ops(cpu);  in smp_cpu_setup()
496 	if (ops->cpu_init(cpu))  in smp_cpu_setup()
499 	set_cpu_possible(cpu, true);  in smp_cpu_setup()
507 int arch_register_cpu(int cpu)  in arch_register_cpu()  argument
509 	acpi_handle acpi_handle = acpi_get_processor_handle(cpu);  in arch_register_cpu()
510 	struct cpu *c = &per_cpu(cpu_devices, cpu);  in arch_register_cpu()
517 	/* For now block anything that looks like physical CPU Hotplug */  in arch_register_cpu()
518 	if (invalid_logical_cpuid(cpu) || !cpu_present(cpu)) {  in arch_register_cpu()
519 		pr_err_once("Changing CPU present bit is not supported\n");  in arch_register_cpu()
528 	c->hotpluggable = arch_cpu_is_hotpluggable(cpu);  in arch_register_cpu()
530 	return register_cpu(c, cpu);  in arch_register_cpu()
534 void arch_unregister_cpu(int cpu)  in arch_unregister_cpu()  argument
536 	acpi_handle acpi_handle = acpi_get_processor_handle(cpu);  in arch_unregister_cpu()
537 	struct cpu *c = &per_cpu(cpu_devices, cpu);  in arch_unregister_cpu()
542 		pr_err_once("Removing a CPU without associated ACPI handle\n");  in arch_unregister_cpu()
550 	/* For now do not allow anything that looks like physical CPU HP */  in arch_unregister_cpu()
551 	if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) {  in arch_unregister_cpu()
552 		pr_err_once("Changing CPU present bit is not supported\n");  in arch_unregister_cpu()
563 struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)  in acpi_cpu_get_madt_gicc()  argument
565 	return &cpu_madt_gicc[cpu];  in acpi_cpu_get_madt_gicc()
582 		pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);  in acpi_map_gic_cpu_interface()
587 		pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);  in acpi_map_gic_cpu_interface()
592 		pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);  in acpi_map_gic_cpu_interface()
596 	/* Check if GICC structure of boot CPU is available in the MADT */  in acpi_map_gic_cpu_interface()
599 			pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",  in acpi_map_gic_cpu_interface()
611 	/* map the logical cpu id to cpu MPIDR */  in acpi_map_gic_cpu_interface()
617 	 * Set-up the ACPI parking protocol cpu entries  in acpi_map_gic_cpu_interface()
622 	 * initialize the cpu if the parking protocol is  in acpi_map_gic_cpu_interface()
660 	 * In ACPI, SMP and CPU NUMA information is provided in separate  in acpi_parse_and_init_cpus()
663 	 * Thus, it is simpler to first create the cpu logical map through  in acpi_parse_and_init_cpus()
677  * Enumerate the possible CPU set from the device tree and build the
678  * cpu logical map array containing MPIDR values related to logical
692 			pr_err("%pOF: duplicate cpu reg properties in the DT\n",  in of_parse_and_init_cpus()
698 		 * The numbering scheme requires that the boot CPU  in of_parse_and_init_cpus()
705 				pr_err("%pOF: duplicate boot cpu reg property in DT\n",  in of_parse_and_init_cpus()
715 			 * initialized and the boot cpu doesn't need  in of_parse_and_init_cpus()
717 			 * incrementing cpu.  in of_parse_and_init_cpus()
725 		pr_debug("cpu logical map 0x%llx\n", hwid);  in of_parse_and_init_cpus()
735  * Enumerate the possible CPU set from the device tree or ACPI and build the
736  * cpu logical map array containing MPIDR values related to logical
753 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");  in smp_init_cpus()
759 	 * the cpus so that cpu processor description entries (DT cpu nodes  in smp_init_cpus()
760 	 * and ACPI MADT entries) can be retrieved by matching the cpu hwid  in smp_init_cpus()
762 	 * If the cpu set-up fails, invalidate the cpu_logical_map entry.  in smp_init_cpus()
776 	unsigned int cpu;  in smp_prepare_cpus()  local
798 	for_each_possible_cpu(cpu) {  in smp_prepare_cpus()
800 		if (cpu == smp_processor_id())  in smp_prepare_cpus()
803 		ops = get_cpu_ops(cpu);  in smp_prepare_cpus()
807 		err = ops->cpu_prepare(cpu);  in smp_prepare_cpus()
811 		set_cpu_present(cpu, true);  in smp_prepare_cpus()
812 		numa_store_cpu_info(cpu);  in smp_prepare_cpus()
819 	[IPI_CPU_STOP]		= "CPU stop interrupts",
820 	[IPI_CPU_STOP_NMI]	= "CPU stop NMIs",
823 	[IPI_CPU_BACKTRACE]	= "CPU backtrace interrupts",
833 	unsigned int cpu, i;  in arch_show_interrupts()  local
838 		for_each_online_cpu(cpu)  in arch_show_interrupts()
839 			seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));  in arch_show_interrupts()
852 void arch_send_call_function_single_ipi(int cpu)  in arch_send_call_function_single_ipi()  argument
854 	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);  in arch_send_call_function_single_ipi()
864 static void __noreturn local_cpu_stop(unsigned int cpu)  in local_cpu_stop()  argument
866 	set_cpu_online(cpu, false);  in local_cpu_stop()
883 static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)  in ipi_cpu_crash_stop()  argument
896 	crash_save_cpu(regs, cpu);  in ipi_cpu_crash_stop()
898 	set_cpu_online(cpu, false);  in ipi_cpu_crash_stop()
903 		__cpu_try_die(cpu);  in ipi_cpu_crash_stop()
914 	unsigned int cpu;  in arm64_send_ipi()  local
919 		for_each_cpu(cpu, mask)  in arm64_send_ipi()
920 			__ipi_send_single(get_ipi_desc(cpu, nr), cpu);  in arm64_send_ipi()
943 	int cpu;  in kgdb_roundup_cpus()  local
945 	for_each_online_cpu(cpu) {  in kgdb_roundup_cpus()
947 		if (cpu == this_cpu)  in kgdb_roundup_cpus()
950 		__ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);  in kgdb_roundup_cpus()
960 	unsigned int cpu = smp_processor_id();  in do_handle_IPI()  local
977 			ipi_cpu_crash_stop(cpu, get_irq_regs());  in do_handle_IPI()
980 			local_cpu_stop(cpu);  in do_handle_IPI()
1005 		kgdb_nmicallback(cpu, get_irq_regs());  in do_handle_IPI()
1009 		pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);  in do_handle_IPI()
1046 static void ipi_setup(int cpu)  in ipi_setup()  argument
1062 			enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));  in ipi_setup()
1068 static void ipi_teardown(int cpu)  in ipi_teardown()  argument
1084 			disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));  in ipi_teardown()
1092 	int err, irq, cpu;  in ipi_setup_sgi()  local
1104 	for_each_possible_cpu(cpu)  in ipi_setup_sgi()
1105 		get_ipi_desc(cpu, ipi) = irq_to_desc(irq);  in ipi_setup_sgi()
1112 	for (int cpu = 0; cpu < ncpus; cpu++) {  in ipi_setup_lpi()  local
1115 		irq = ipi_irq_base + (cpu * nr_ipi) + ipi;  in ipi_setup_lpi()
1117 		err = irq_force_affinity(irq, cpumask_of(cpu));  in ipi_setup_lpi()
1126 		get_ipi_desc(cpu, ipi) = irq_to_desc(irq);  in ipi_setup_lpi()
1147 	/* Setup the boot CPU immediately */  in set_smp_ipi_range_percpu()
1151 void arch_smp_send_reschedule(int cpu)  in arch_smp_send_reschedule()  argument
1153 	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);  in arch_smp_send_reschedule()
1157 void arch_send_wakeup_ipi(unsigned int cpu)  in arch_send_wakeup_ipi()  argument
1160 	 * We use a scheduler IPI to wake the CPU as this avoids the need for a  in arch_send_wakeup_ipi()
1163 	smp_send_reschedule(cpu);  in arch_send_wakeup_ipi()
1175  * The number of CPUs online, not counting this CPU (which may not be
1192 	 * If this cpu is the only one alive at this point in time, online or  in smp_send_stop()
1198 	/* Only proceed if this is the first CPU to reach this code */  in smp_send_stop()
1203 	 * Send an IPI to all currently online CPUs except the CPU running  in smp_send_stop()
1210 	 * grab the CPU hotplug mutex ourselves. Worst case:  in smp_send_stop()
1211 	 * - If a CPU comes online as we're running, we'll likely notice it  in smp_send_stop()
1218 	 *   the fact that there could be cases where a CPU can't be stopped.  in smp_send_stop()