1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include <linux/cpumask.h>
4 #include <linux/delay.h>
5 #include <linux/smp.h>
6 #include <linux/string_choices.h>
7 
8 #include <asm/io_apic.h>
9 
10 #include "local.h"
11 
12 DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
13 
14 #ifdef CONFIG_SMP
15 static int apic_ipi_shorthand_off __ro_after_init;
16 
apic_ipi_shorthand(char * str)17 static __init int apic_ipi_shorthand(char *str)
18 {
19 	get_option(&str, &apic_ipi_shorthand_off);
20 	return 1;
21 }
22 __setup("no_ipi_broadcast=", apic_ipi_shorthand);
23 
print_ipi_mode(void)24 static int __init print_ipi_mode(void)
25 {
26 	pr_info("IPI shorthand broadcast: %s\n",
27 		str_disabled_enabled(apic_ipi_shorthand_off));
28 	return 0;
29 }
30 late_initcall(print_ipi_mode);
31 
apic_smt_update(void)32 void apic_smt_update(void)
33 {
34 	/*
35 	 * Do not switch to broadcast mode if:
36 	 * - Disabled on the command line
37 	 * - Only a single CPU is online
38 	 * - Not all present CPUs have been at least booted once
39 	 *
40 	 * The latter is important as the local APIC might be in some
41 	 * random state and a broadcast might cause havoc. That's
42 	 * especially true for NMI broadcasting.
43 	 */
44 	if (apic_ipi_shorthand_off || num_online_cpus() == 1 ||
45 	    !cpumask_equal(cpu_present_mask, &cpus_booted_once_mask)) {
46 		static_branch_disable(&apic_use_ipi_shorthand);
47 	} else {
48 		static_branch_enable(&apic_use_ipi_shorthand);
49 	}
50 }
51 
apic_send_IPI_allbutself(unsigned int vector)52 void apic_send_IPI_allbutself(unsigned int vector)
53 {
54 	if (num_online_cpus() < 2)
55 		return;
56 
57 	if (static_branch_likely(&apic_use_ipi_shorthand))
58 		__apic_send_IPI_allbutself(vector);
59 	else
60 		__apic_send_IPI_mask_allbutself(cpu_online_mask, vector);
61 }
62 
63 /*
64  * Send a 'reschedule' IPI to another CPU. It goes straight through and
65  * wastes no time serializing anything. Worst case is that we lose a
66  * reschedule ...
67  */
native_smp_send_reschedule(int cpu)68 void native_smp_send_reschedule(int cpu)
69 {
70 	if (unlikely(cpu_is_offline(cpu))) {
71 		WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
72 		return;
73 	}
74 	__apic_send_IPI(cpu, RESCHEDULE_VECTOR);
75 }
76 
native_send_call_func_single_ipi(int cpu)77 void native_send_call_func_single_ipi(int cpu)
78 {
79 	__apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
80 }
81 
native_send_call_func_ipi(const struct cpumask * mask)82 void native_send_call_func_ipi(const struct cpumask *mask)
83 {
84 	if (static_branch_likely(&apic_use_ipi_shorthand)) {
85 		unsigned int cpu = smp_processor_id();
86 
87 		if (!cpumask_or_equal(mask, cpumask_of(cpu), cpu_online_mask))
88 			goto sendmask;
89 
90 		if (cpumask_test_cpu(cpu, mask))
91 			__apic_send_IPI_all(CALL_FUNCTION_VECTOR);
92 		else if (num_online_cpus() > 1)
93 			__apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR);
94 		return;
95 	}
96 
97 sendmask:
98 	__apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
99 }
100 
apic_send_nmi_to_offline_cpu(unsigned int cpu)101 void apic_send_nmi_to_offline_cpu(unsigned int cpu)
102 {
103 	if (WARN_ON_ONCE(!apic->nmi_to_offline_cpu))
104 		return;
105 	if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, &cpus_booted_once_mask)))
106 		return;
107 	apic->send_IPI(cpu, NMI_VECTOR);
108 }
109 #endif /* CONFIG_SMP */
110 
__prepare_ICR2(unsigned int mask)111 static inline int __prepare_ICR2(unsigned int mask)
112 {
113 	return SET_XAPIC_DEST_FIELD(mask);
114 }
115 
apic_mem_wait_icr_idle_timeout(void)116 u32 apic_mem_wait_icr_idle_timeout(void)
117 {
118 	int cnt;
119 
120 	for (cnt = 0; cnt < 1000; cnt++) {
121 		if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
122 			return 0;
123 		inc_irq_stat(icr_read_retry_count);
124 		udelay(100);
125 	}
126 	return APIC_ICR_BUSY;
127 }
128 
apic_mem_wait_icr_idle(void)129 void apic_mem_wait_icr_idle(void)
130 {
131 	while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
132 		cpu_relax();
133 }
134 
135 /*
136  * This is safe against interruption because it only writes the lower 32
137  * bits of the APIC_ICR register. The destination field is ignored for
138  * short hand IPIs.
139  *
140  *  wait_icr_idle()
141  *  write(ICR2, dest)
142  *  NMI
143  *	wait_icr_idle()
144  *	write(ICR)
145  *	wait_icr_idle()
146  *  write(ICR)
147  *
148  * This function does not need to disable interrupts as there is no ICR2
149  * interaction. The memory write is direct except when the machine is
150  * affected by the 11AP Pentium erratum, which turns the plain write into
151  * an XCHG operation.
152  */
__default_send_IPI_shortcut(unsigned int shortcut,int vector)153 static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
154 {
155 	/*
156 	 * Wait for the previous ICR command to complete.  Use
157 	 * safe_apic_wait_icr_idle() for the NMI vector as there have been
158 	 * issues where otherwise the system hangs when the panic CPU tries
159 	 * to stop the others before launching the kdump kernel.
160 	 */
161 	if (unlikely(vector == NMI_VECTOR))
162 		apic_mem_wait_icr_idle_timeout();
163 	else
164 		apic_mem_wait_icr_idle();
165 
166 	/* Destination field (ICR2) and the destination mode are ignored */
167 	native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0));
168 }
169 
170 /*
171  * This is used to send an IPI with no shorthand notation (the destination is
172  * specified in bits 56 to 63 of the ICR).
173  */
__default_send_IPI_dest_field(unsigned int dest_mask,int vector,unsigned int dest_mode)174 void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
175 				   unsigned int dest_mode)
176 {
177 	/* See comment in __default_send_IPI_shortcut() */
178 	if (unlikely(vector == NMI_VECTOR))
179 		apic_mem_wait_icr_idle_timeout();
180 	else
181 		apic_mem_wait_icr_idle();
182 
183 	/* Set the IPI destination field in the ICR */
184 	native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask));
185 	/* Send it with the proper destination mode */
186 	native_apic_mem_write(APIC_ICR, __prepare_ICR(0, vector, dest_mode));
187 }
188 
default_send_IPI_single_phys(int cpu,int vector)189 void default_send_IPI_single_phys(int cpu, int vector)
190 {
191 	unsigned long flags;
192 
193 	local_irq_save(flags);
194 	__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
195 				      vector, APIC_DEST_PHYSICAL);
196 	local_irq_restore(flags);
197 }
198 
default_send_IPI_mask_sequence_phys(const struct cpumask * mask,int vector)199 void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
200 {
201 	unsigned long flags;
202 	unsigned long cpu;
203 
204 	local_irq_save(flags);
205 	for_each_cpu(cpu, mask) {
206 		__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
207 				cpu), vector, APIC_DEST_PHYSICAL);
208 	}
209 	local_irq_restore(flags);
210 }
211 
default_send_IPI_mask_allbutself_phys(const struct cpumask * mask,int vector)212 void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
213 						 int vector)
214 {
215 	unsigned int cpu, this_cpu = smp_processor_id();
216 	unsigned long flags;
217 
218 	local_irq_save(flags);
219 	for_each_cpu(cpu, mask) {
220 		if (cpu == this_cpu)
221 			continue;
222 		__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
223 				 cpu), vector, APIC_DEST_PHYSICAL);
224 	}
225 	local_irq_restore(flags);
226 }
227 
228 /*
229  * Helper function for APICs which insist on cpumasks
230  */
default_send_IPI_single(int cpu,int vector)231 void default_send_IPI_single(int cpu, int vector)
232 {
233 	__apic_send_IPI_mask(cpumask_of(cpu), vector);
234 }
235 
default_send_IPI_allbutself(int vector)236 void default_send_IPI_allbutself(int vector)
237 {
238 	__default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
239 }
240 
default_send_IPI_all(int vector)241 void default_send_IPI_all(int vector)
242 {
243 	__default_send_IPI_shortcut(APIC_DEST_ALLINC, vector);
244 }
245 
default_send_IPI_self(int vector)246 void default_send_IPI_self(int vector)
247 {
248 	__default_send_IPI_shortcut(APIC_DEST_SELF, vector);
249 }
250 
251 #ifdef CONFIG_X86_32
default_send_IPI_mask_sequence_logical(const struct cpumask * mask,int vector)252 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
253 {
254 	unsigned long flags;
255 	unsigned int cpu;
256 
257 	local_irq_save(flags);
258 	for_each_cpu(cpu, mask)
259 		__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
260 	local_irq_restore(flags);
261 }
262 
default_send_IPI_mask_allbutself_logical(const struct cpumask * mask,int vector)263 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
264 						 int vector)
265 {
266 	unsigned int cpu, this_cpu = smp_processor_id();
267 	unsigned long flags;
268 
269 	local_irq_save(flags);
270 	for_each_cpu(cpu, mask) {
271 		if (cpu == this_cpu)
272 			continue;
273 		__default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
274 	}
275 	local_irq_restore(flags);
276 }
277 
default_send_IPI_mask_logical(const struct cpumask * cpumask,int vector)278 void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
279 {
280 	unsigned long mask = cpumask_bits(cpumask)[0];
281 	unsigned long flags;
282 
283 	if (!mask)
284 		return;
285 
286 	local_irq_save(flags);
287 	WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
288 	__default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
289 	local_irq_restore(flags);
290 }
291 #endif
292