1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/cpumask.h>
4 #include <linux/delay.h>
5 #include <linux/smp.h>
6 #include <linux/string_choices.h>
7
8 #include <asm/io_apic.h>
9
10 #include "local.h"
11
12 DEFINE_STATIC_KEY_FALSE(apic_use_ipi_shorthand);
13
14 #ifdef CONFIG_SMP
15 static int apic_ipi_shorthand_off __ro_after_init;
16
apic_ipi_shorthand(char * str)17 static __init int apic_ipi_shorthand(char *str)
18 {
19 get_option(&str, &apic_ipi_shorthand_off);
20 return 1;
21 }
22 __setup("no_ipi_broadcast=", apic_ipi_shorthand);
23
print_ipi_mode(void)24 static int __init print_ipi_mode(void)
25 {
26 pr_info("IPI shorthand broadcast: %s\n",
27 str_disabled_enabled(apic_ipi_shorthand_off));
28 return 0;
29 }
30 late_initcall(print_ipi_mode);
31
apic_smt_update(void)32 void apic_smt_update(void)
33 {
34 /*
35 * Do not switch to broadcast mode if:
36 * - Disabled on the command line
37 * - Only a single CPU is online
38 * - Not all present CPUs have been at least booted once
39 *
40 * The latter is important as the local APIC might be in some
41 * random state and a broadcast might cause havoc. That's
42 * especially true for NMI broadcasting.
43 */
44 if (apic_ipi_shorthand_off || num_online_cpus() == 1 ||
45 !cpumask_equal(cpu_present_mask, &cpus_booted_once_mask)) {
46 static_branch_disable(&apic_use_ipi_shorthand);
47 } else {
48 static_branch_enable(&apic_use_ipi_shorthand);
49 }
50 }
51
apic_send_IPI_allbutself(unsigned int vector)52 void apic_send_IPI_allbutself(unsigned int vector)
53 {
54 if (num_online_cpus() < 2)
55 return;
56
57 if (static_branch_likely(&apic_use_ipi_shorthand))
58 __apic_send_IPI_allbutself(vector);
59 else
60 __apic_send_IPI_mask_allbutself(cpu_online_mask, vector);
61 }
62
63 /*
64 * Send a 'reschedule' IPI to another CPU. It goes straight through and
65 * wastes no time serializing anything. Worst case is that we lose a
66 * reschedule ...
67 */
native_smp_send_reschedule(int cpu)68 void native_smp_send_reschedule(int cpu)
69 {
70 if (unlikely(cpu_is_offline(cpu))) {
71 WARN(1, "sched: Unexpected reschedule of offline CPU#%d!\n", cpu);
72 return;
73 }
74 __apic_send_IPI(cpu, RESCHEDULE_VECTOR);
75 }
76
native_send_call_func_single_ipi(int cpu)77 void native_send_call_func_single_ipi(int cpu)
78 {
79 __apic_send_IPI(cpu, CALL_FUNCTION_SINGLE_VECTOR);
80 }
81
native_send_call_func_ipi(const struct cpumask * mask)82 void native_send_call_func_ipi(const struct cpumask *mask)
83 {
84 if (static_branch_likely(&apic_use_ipi_shorthand)) {
85 unsigned int cpu = smp_processor_id();
86
87 if (!cpumask_or_equal(mask, cpumask_of(cpu), cpu_online_mask))
88 goto sendmask;
89
90 if (cpumask_test_cpu(cpu, mask))
91 __apic_send_IPI_all(CALL_FUNCTION_VECTOR);
92 else if (num_online_cpus() > 1)
93 __apic_send_IPI_allbutself(CALL_FUNCTION_VECTOR);
94 return;
95 }
96
97 sendmask:
98 __apic_send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
99 }
100
apic_send_nmi_to_offline_cpu(unsigned int cpu)101 void apic_send_nmi_to_offline_cpu(unsigned int cpu)
102 {
103 if (WARN_ON_ONCE(!apic->nmi_to_offline_cpu))
104 return;
105 if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, &cpus_booted_once_mask)))
106 return;
107 apic->send_IPI(cpu, NMI_VECTOR);
108 }
109 #endif /* CONFIG_SMP */
110
__prepare_ICR2(unsigned int mask)111 static inline int __prepare_ICR2(unsigned int mask)
112 {
113 return SET_XAPIC_DEST_FIELD(mask);
114 }
115
apic_mem_wait_icr_idle_timeout(void)116 u32 apic_mem_wait_icr_idle_timeout(void)
117 {
118 int cnt;
119
120 for (cnt = 0; cnt < 1000; cnt++) {
121 if (!(apic_read(APIC_ICR) & APIC_ICR_BUSY))
122 return 0;
123 inc_irq_stat(icr_read_retry_count);
124 udelay(100);
125 }
126 return APIC_ICR_BUSY;
127 }
128
apic_mem_wait_icr_idle(void)129 void apic_mem_wait_icr_idle(void)
130 {
131 while (native_apic_mem_read(APIC_ICR) & APIC_ICR_BUSY)
132 cpu_relax();
133 }
134
135 /*
136 * This is safe against interruption because it only writes the lower 32
137 * bits of the APIC_ICR register. The destination field is ignored for
138 * short hand IPIs.
139 *
140 * wait_icr_idle()
141 * write(ICR2, dest)
142 * NMI
143 * wait_icr_idle()
144 * write(ICR)
145 * wait_icr_idle()
146 * write(ICR)
147 *
148 * This function does not need to disable interrupts as there is no ICR2
149 * interaction. The memory write is direct except when the machine is
150 * affected by the 11AP Pentium erratum, which turns the plain write into
151 * an XCHG operation.
152 */
__default_send_IPI_shortcut(unsigned int shortcut,int vector)153 static void __default_send_IPI_shortcut(unsigned int shortcut, int vector)
154 {
155 /*
156 * Wait for the previous ICR command to complete. Use
157 * safe_apic_wait_icr_idle() for the NMI vector as there have been
158 * issues where otherwise the system hangs when the panic CPU tries
159 * to stop the others before launching the kdump kernel.
160 */
161 if (unlikely(vector == NMI_VECTOR))
162 apic_mem_wait_icr_idle_timeout();
163 else
164 apic_mem_wait_icr_idle();
165
166 /* Destination field (ICR2) and the destination mode are ignored */
167 native_apic_mem_write(APIC_ICR, __prepare_ICR(shortcut, vector, 0));
168 }
169
170 /*
171 * This is used to send an IPI with no shorthand notation (the destination is
172 * specified in bits 56 to 63 of the ICR).
173 */
__default_send_IPI_dest_field(unsigned int dest_mask,int vector,unsigned int dest_mode)174 void __default_send_IPI_dest_field(unsigned int dest_mask, int vector,
175 unsigned int dest_mode)
176 {
177 /* See comment in __default_send_IPI_shortcut() */
178 if (unlikely(vector == NMI_VECTOR))
179 apic_mem_wait_icr_idle_timeout();
180 else
181 apic_mem_wait_icr_idle();
182
183 /* Set the IPI destination field in the ICR */
184 native_apic_mem_write(APIC_ICR2, __prepare_ICR2(dest_mask));
185 /* Send it with the proper destination mode */
186 native_apic_mem_write(APIC_ICR, __prepare_ICR(0, vector, dest_mode));
187 }
188
default_send_IPI_single_phys(int cpu,int vector)189 void default_send_IPI_single_phys(int cpu, int vector)
190 {
191 unsigned long flags;
192
193 local_irq_save(flags);
194 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu),
195 vector, APIC_DEST_PHYSICAL);
196 local_irq_restore(flags);
197 }
198
default_send_IPI_mask_sequence_phys(const struct cpumask * mask,int vector)199 void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
200 {
201 unsigned long flags;
202 unsigned long cpu;
203
204 local_irq_save(flags);
205 for_each_cpu(cpu, mask) {
206 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
207 cpu), vector, APIC_DEST_PHYSICAL);
208 }
209 local_irq_restore(flags);
210 }
211
default_send_IPI_mask_allbutself_phys(const struct cpumask * mask,int vector)212 void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
213 int vector)
214 {
215 unsigned int cpu, this_cpu = smp_processor_id();
216 unsigned long flags;
217
218 local_irq_save(flags);
219 for_each_cpu(cpu, mask) {
220 if (cpu == this_cpu)
221 continue;
222 __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
223 cpu), vector, APIC_DEST_PHYSICAL);
224 }
225 local_irq_restore(flags);
226 }
227
228 /*
229 * Helper function for APICs which insist on cpumasks
230 */
default_send_IPI_single(int cpu,int vector)231 void default_send_IPI_single(int cpu, int vector)
232 {
233 __apic_send_IPI_mask(cpumask_of(cpu), vector);
234 }
235
default_send_IPI_allbutself(int vector)236 void default_send_IPI_allbutself(int vector)
237 {
238 __default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
239 }
240
default_send_IPI_all(int vector)241 void default_send_IPI_all(int vector)
242 {
243 __default_send_IPI_shortcut(APIC_DEST_ALLINC, vector);
244 }
245
default_send_IPI_self(int vector)246 void default_send_IPI_self(int vector)
247 {
248 __default_send_IPI_shortcut(APIC_DEST_SELF, vector);
249 }
250
251 #ifdef CONFIG_X86_32
default_send_IPI_mask_sequence_logical(const struct cpumask * mask,int vector)252 void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, int vector)
253 {
254 unsigned long flags;
255 unsigned int cpu;
256
257 local_irq_save(flags);
258 for_each_cpu(cpu, mask)
259 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
260 local_irq_restore(flags);
261 }
262
default_send_IPI_mask_allbutself_logical(const struct cpumask * mask,int vector)263 void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
264 int vector)
265 {
266 unsigned int cpu, this_cpu = smp_processor_id();
267 unsigned long flags;
268
269 local_irq_save(flags);
270 for_each_cpu(cpu, mask) {
271 if (cpu == this_cpu)
272 continue;
273 __default_send_IPI_dest_field(1U << cpu, vector, APIC_DEST_LOGICAL);
274 }
275 local_irq_restore(flags);
276 }
277
default_send_IPI_mask_logical(const struct cpumask * cpumask,int vector)278 void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
279 {
280 unsigned long mask = cpumask_bits(cpumask)[0];
281 unsigned long flags;
282
283 if (!mask)
284 return;
285
286 local_irq_save(flags);
287 WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
288 __default_send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
289 local_irq_restore(flags);
290 }
291 #endif
292