1 // SPDX-License-Identifier: GPL-2.0
8  * print a warning if not and turn off the TSC clock-source.
10  * The warp-check is point-to-point between two CPUs, the CPU
14  * Only two CPUs may participate - they can enter in any order.
35  * TSC's on different sockets may be reset asynchronously.
36  * This may cause the TSC ADJUST value on socket 0 to be NOT 0.
48 void tsc_verify_tsc_adjust(bool resume)  in tsc_verify_tsc_adjust()  argument
61 	if (!resume && time_before(jiffies, adj->nextcheck))  in tsc_verify_tsc_adjust()
64 	adj->nextcheck = jiffies + HZ;  in tsc_verify_tsc_adjust()
67 	if (adj->adjusted == curval)  in tsc_verify_tsc_adjust()
71 	wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted);  in tsc_verify_tsc_adjust()
73 	if (!adj->warned || resume) {  in tsc_verify_tsc_adjust()
74 		pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n",  in tsc_verify_tsc_adjust()
75 			smp_processor_id(), adj->adjusted, curval);  in tsc_verify_tsc_adjust()
76 		adj->warned = true;  in tsc_verify_tsc_adjust()
89 	 * On the boot cpu we just force set the ADJUST value to 0 if it's  in tsc_sanitize_first_cpu()
90 	 * non zero. We don't do that on non boot cpus because physical  in tsc_sanitize_first_cpu()
96 	 * when multiple sockets are reset asynchronously with each other  in tsc_sanitize_first_cpu()
110 	cur->adjusted = bootval;  in tsc_sanitize_first_cpu()
127 	cur->bootval = bootval;  in tsc_store_and_check_tsc_adjust()
128 	cur->nextcheck = jiffies + HZ;  in tsc_store_and_check_tsc_adjust()
149 	cur->bootval = bootval;  in tsc_store_and_check_tsc_adjust()
150 	cur->nextcheck = jiffies + HZ;  in tsc_store_and_check_tsc_adjust()
151 	cur->warned = false;  in tsc_store_and_check_tsc_adjust()
154 	 * If a non-zero TSC value for socket 0 may be valid then the default  in tsc_store_and_check_tsc_adjust()
158 		cur->adjusted = bootval;  in tsc_store_and_check_tsc_adjust()
164 	 * where TSC_ADJUST is expected to be different. When called on the  in tsc_store_and_check_tsc_adjust()
181 	if (bootval != ref->bootval)  in tsc_store_and_check_tsc_adjust()
186 	 * value on this newly upcoming CPU differs from the adjustment  in tsc_store_and_check_tsc_adjust()
190 	if (bootval != ref->adjusted) {  in tsc_store_and_check_tsc_adjust()
191 		cur->adjusted = ref->adjusted;  in tsc_store_and_check_tsc_adjust()
192 		wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted);  in tsc_store_and_check_tsc_adjust()
195 	 * We have the TSCs forced to be in sync on this package. Skip sync  in tsc_store_and_check_tsc_adjust()
212  * we want to have the fastest, inlined, non-debug version
213  * of a critical section, to be able to prove TSC time-warps:
223  * TSC-warp measurement loop running on both CPUs.  This is not called
240 		 * previous TSC that was measured (possibly on  in check_tsc_warp()
263 		 * we saw a time-warp of the TSC going backwards:  in check_tsc_warp()
267 			max_warp = max(max_warp, prev - now);  in check_tsc_warp()
280 	WARN(!(now-start),  in check_tsc_warp()
282 			now-start, end-start);  in check_tsc_warp()
287  * If the target CPU coming online doesn't have any of its core-siblings
288  * online, a timeout of 20msec will be used for the TSC-warp measurement
291  * have more and more logical-siblings in that socket).
293  * Ideally we should be able to skip the TSC sync check on the other
294  * core-siblings, if the first logical CPU in a socket passed the sync test.
295  * But as the TSC is per-logical CPU and can potentially be modified wrongly
298  * cores in the socket don't get reset at the same time.
306  * Source CPU calls into this - it waits for the freshly booted
333 	while (atomic_read(&start_count) != cpus - 1) {  in check_tsc_sync_source()
348 	while (atomic_read(&stop_count) != cpus-1)  in check_tsc_sync_source()
359 		pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",  in check_tsc_sync_source()
366 		pr_warn("TSC synchronization [CPU#%d -> CPU#%d]:\n",  in check_tsc_sync_source()
376 	 * Reset it - just in case we boot another CPU later:  in check_tsc_sync_source()
415 	 * is true for SoCs which have no fallback clocksource. On these  in check_tsc_sync_target()
452 	 * Reset it for the next sync test:  in check_tsc_sync_target()
467 	 * needs to move backwards.  in check_tsc_sync_target()
470 		cur_max_warp = -gbl_max_warp;  in check_tsc_sync_target()
477 	 * really depends on CPU, node distance and frequency. So  in check_tsc_sync_target()
480 	 * value is used. In the worst case the adjustment needs to go  in check_tsc_sync_target()
483 	cur->adjusted += cur_max_warp;  in check_tsc_sync_target()
486 		cpu, cur_max_warp, cur->adjusted);  in check_tsc_sync_target()
488 	wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted);  in check_tsc_sync_target()