1 // SPDX-License-Identifier: GPL-2.0+
15 #include <linux/tick.h>
20 #include "tick-internal.h"
24  * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
29  * @maxsec:	guaranteed runtime conversion range in seconds
34  * @to and @from are frequency values in HZ. For clock sources @to is
38  * The @maxsec conversion range argument controls the time frame in
40  * calculated mult and shift factors. This guarantees that no 64bit
59 		sftacc--;  in clocks_calc_mult_shift()
66 	for (sft = 32; sft > 0; sft--) {  in clocks_calc_mult_shift()
78 /*[Clocksource internal variables]---------
82  *	used to calculate the suspend time.
88  *	Name of the user-specified clocksource.
106  * Also a default for cs->uncertainty_margin when registering clocks.
114  * a lower bound for cs->uncertainty_margin values when registering clocks.
162 	 * of the kernel, so there is no point in creating a more permanent  in clocksource_watchdog_work()
173 	cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);  in __clocksource_unstable()
174 	cs->flags |= CLOCK_SOURCE_UNSTABLE;  in __clocksource_unstable()
178 	 * re-rate and re-select.  in __clocksource_unstable()
180 	if (list_empty(&cs->list)) {  in __clocksource_unstable()
181 		cs->rating = 0;  in __clocksource_unstable()
185 	if (cs->mark_unstable)  in __clocksource_unstable()
186 		cs->mark_unstable(cs);  in __clocksource_unstable()
194  * clocksource_mark_unstable - mark clocksource unstable via watchdog
198  * it defers demotion and re-selection to a kthread.
205 	if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {  in clocksource_mark_unstable()
206 		if (!list_empty(&cs->list) && list_empty(&cs->wd_list))  in clocksource_mark_unstable()
207 			list_add(&cs->wd_list, &watchdog_list);  in clocksource_mark_unstable()
233 		*wdnow = watchdog->read(watchdog);  in cs_watchdog_read()
234 		*csnow = cs->read(cs);  in cs_watchdog_read()
235 		wd_end = watchdog->read(watchdog);  in cs_watchdog_read()
236 		wd_end2 = watchdog->read(watchdog);  in cs_watchdog_read()
239 		wd_delta = clocksource_delta(wd_end, *wdnow, watchdog->mask);  in cs_watchdog_read()
240 		wd_delay = clocksource_cyc2ns(wd_delta, watchdog->mult,  in cs_watchdog_read()
241 					      watchdog->shift);  in cs_watchdog_read()
245 					smp_processor_id(), watchdog->name, nretries);  in cs_watchdog_read()
251 		 * Now compute delay in consecutive watchdog read to see if  in cs_watchdog_read()
253 		 * significant delay in reading both clocksource and watchdog.  in cs_watchdog_read()
255 		 * If consecutive WD read-back delay > WATCHDOG_MAX_SKEW/2,  in cs_watchdog_read()
259 		wd_delta = clocksource_delta(wd_end2, wd_end, watchdog->mask);  in cs_watchdog_read()
260 		wd_seq_delay = clocksource_cyc2ns(wd_delta, watchdog->mult, watchdog->shift);  in cs_watchdog_read()
265 …warn("timekeeping watchdog on CPU%d: wd-%s-wd excessive read-back delay of %lldns vs. limit of %ld…  in cs_watchdog_read()
266 		smp_processor_id(), cs->name, wd_delay, WATCHDOG_MAX_SKEW, wd_seq_delay, nretries, cs->name);  in cs_watchdog_read()
270 	pr_info("timekeeping watchdog on CPU%d: %s wd-wd read-back delay of %lldns\n",  in cs_watchdog_read()
271 		smp_processor_id(), watchdog->name, wd_seq_delay);  in cs_watchdog_read()
272 	pr_info("wd-%s-wd read-back delay of %lldns, clock-skew test skipped!\n",  in cs_watchdog_read()
273 		cs->name, wd_delay);  in cs_watchdog_read()
293 	/* If no checking desired, or no other CPU to check, leave. */  in clocksource_verify_choose_cpus()
313 	 * and no replacement CPU is selected.  This gracefully handles  in clocksource_verify_choose_cpus()
319 		cpu = cpumask_next(cpu - 1, cpu_online_mask);  in clocksource_verify_choose_cpus()
334 	csnow_mid = cs->read(cs);  in clocksource_verify_one_cpu()
354 		pr_warn("Not enough CPUs to check clocksource '%s'.\n", cs->name);  in clocksource_verify_percpu()
358 …pr_warn("Checking clocksource %s synchronization from CPU %d to CPUs %*pbl.\n", cs->name, testcpu,…  in clocksource_verify_percpu()
362 		csnow_begin = cs->read(cs);  in clocksource_verify_percpu()
364 		csnow_end = cs->read(cs);  in clocksource_verify_percpu()
365 		delta = (s64)((csnow_mid - csnow_begin) & cs->mask);  in clocksource_verify_percpu()
368 		delta = (csnow_end - csnow_mid) & cs->mask;  in clocksource_verify_percpu()
371 		delta = clocksource_delta(csnow_end, csnow_begin, cs->mask);  in clocksource_verify_percpu()
372 		cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);  in clocksource_verify_percpu()
382 			cpumask_pr_args(&cpus_ahead), testcpu, cs->name);  in clocksource_verify_percpu()
385 			cpumask_pr_args(&cpus_behind), testcpu, cs->name);  in clocksource_verify_percpu()
387 		pr_warn("        CPU %d check durations %lldns - %lldns for clocksource %s.\n",  in clocksource_verify_percpu()
388 			testcpu, cs_nsec_min, cs_nsec_max, cs->name);  in clocksource_verify_percpu()
397 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_reset_watchdog()
420 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in clocksource_watchdog()
438 		 * watchdog check. So give system some space and suspend the  in clocksource_watchdog()
444 			 * cs->last could keep unchanged for 5 minutes, reset  in clocksource_watchdog()
453 		if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||  in clocksource_watchdog()
455 			cs->flags |= CLOCK_SOURCE_WATCHDOG;  in clocksource_watchdog()
456 			cs->wd_last = wdnow;  in clocksource_watchdog()
457 			cs->cs_last = csnow;  in clocksource_watchdog()
461 		delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);  in clocksource_watchdog()
462 		wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,  in clocksource_watchdog()
463 					     watchdog->shift);  in clocksource_watchdog()
465 		delta = clocksource_delta(csnow, cs->cs_last, cs->mask);  in clocksource_watchdog()
466 		cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);  in clocksource_watchdog()
467 		wdlast = cs->wd_last; /* save these in case we print them */  in clocksource_watchdog()
468 		cslast = cs->cs_last;  in clocksource_watchdog()
469 		cs->cs_last = csnow;  in clocksource_watchdog()
470 		cs->wd_last = wdnow;  in clocksource_watchdog()
477 		 * on account of ksoftirqd not getting to run in a timely  in clocksource_watchdog()
497 		md = cs->uncertainty_margin + watchdog->uncertainty_margin;  in clocksource_watchdog()
498 		if (abs(cs_nsec - wd_nsec) > md) {  in clocksource_watchdog()
504 				smp_processor_id(), cs->name);  in clocksource_watchdog()
506 				watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask);  in clocksource_watchdog()
508 				cs->name, cs_nsec, csnow, cslast, cs->mask);  in clocksource_watchdog()
509 			cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem);  in clocksource_watchdog()
512 				cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec);  in clocksource_watchdog()
514 				pr_warn("                      '%s' is current clocksource.\n", cs->name);  in clocksource_watchdog()
516 …                     '%s' (not '%s') is current clocksource.\n", curr_clocksource->name, cs->name);  in clocksource_watchdog()
518 				pr_warn("                      No current clocksource.\n");  in clocksource_watchdog()
523 		if (cs == curr_clocksource && cs->tick_stable)  in clocksource_watchdog()
524 			cs->tick_stable(cs);  in clocksource_watchdog()
526 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&  in clocksource_watchdog()
527 		    (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&  in clocksource_watchdog()
528 		    (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {  in clocksource_watchdog()
529 			/* Mark it valid for high-res. */  in clocksource_watchdog()
530 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_watchdog()
544 			 * clocksource let the tick code know about  in clocksource_watchdog()
548 				cs->flags |= CLOCK_SOURCE_RESELECT;  in clocksource_watchdog()
608 	INIT_LIST_HEAD(&cs->wd_list);  in clocksource_enqueue_watchdog()
610 	if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_enqueue_watchdog()
612 		list_add(&cs->wd_list, &watchdog_list);  in clocksource_enqueue_watchdog()
613 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_enqueue_watchdog()
616 		if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
617 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
634 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)  in clocksource_select_watchdog()
642 		if (!watchdog || cs->rating > watchdog->rating)  in clocksource_select_watchdog()
661 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_dequeue_watchdog()
663 			list_del_init(&cs->wd_list);  in clocksource_dequeue_watchdog()
676 	/* Do any required per-CPU skew verification. */  in __clocksource_watchdog_kthread()
678 	    curr_clocksource->flags & CLOCK_SOURCE_UNSTABLE &&  in __clocksource_watchdog_kthread()
679 	    curr_clocksource->flags & CLOCK_SOURCE_VERIFY_PERCPU)  in __clocksource_watchdog_kthread()
684 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in __clocksource_watchdog_kthread()
685 			list_del_init(&cs->wd_list);  in __clocksource_watchdog_kthread()
689 		if (cs->flags & CLOCK_SOURCE_RESELECT) {  in __clocksource_watchdog_kthread()
690 			cs->flags &= ~CLOCK_SOURCE_RESELECT;  in __clocksource_watchdog_kthread()
719 	if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
720 		cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
743 	 * Skip the clocksource which will be stopped in suspend state.  in __clocksource_suspend_select()
745 	if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP))  in __clocksource_suspend_select()
749 	 * The nonstop clocksource can be selected as the suspend clocksource to  in __clocksource_suspend_select()
750 	 * calculate the suspend time, so it should not supply suspend/resume  in __clocksource_suspend_select()
751 	 * interfaces to suspend the nonstop clocksource when system suspends.  in __clocksource_suspend_select()
753 	if (cs->suspend || cs->resume) {  in __clocksource_suspend_select()
754 		pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n",  in __clocksource_suspend_select()
755 			cs->name);  in __clocksource_suspend_select()
759 	if (!suspend_clocksource || cs->rating > suspend_clocksource->rating)  in __clocksource_suspend_select()
764  * clocksource_suspend_select - Select the best clocksource for suspend timing
785  * clocksource_start_suspend_timing - Start measuring the suspend timing
789  * This function will save the start cycle values of suspend timer to calculate
790  * the suspend time when resuming system.
792  * This function is called late in the suspend process from timekeeping_suspend(),
793  * that means processes are frozen, non-boot cpus and interrupts are disabled
794  * now. It is therefore possible to start the suspend timer without taking the
803 	 * If current clocksource is the suspend timer, we should use the  in clocksource_start_suspend_timing()
805 	 * from suspend timer.  in clocksource_start_suspend_timing()
812 	if (suspend_clocksource->enable &&  in clocksource_start_suspend_timing()
813 	    suspend_clocksource->enable(suspend_clocksource)) {  in clocksource_start_suspend_timing()
814 		pr_warn_once("Failed to enable the non-suspend-able clocksource.\n");  in clocksource_start_suspend_timing()
818 	suspend_start = suspend_clocksource->read(suspend_clocksource);  in clocksource_start_suspend_timing()
822  * clocksource_stop_suspend_timing - Stop measuring the suspend timing
826  * This function will calculate the suspend time from suspend timer.
828  * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource.
830  * This function is called early in the resume process from timekeeping_resume(),
831  * that means there is only one cpu, no processes are running and the interrupts
832  * are disabled. It is therefore possible to stop the suspend timer without
843 	 * If current clocksource is the suspend timer, we should use the  in clocksource_stop_suspend_timing()
845 	 * avoid same reading from suspend timer.  in clocksource_stop_suspend_timing()
850 		now = suspend_clocksource->read(suspend_clocksource);  in clocksource_stop_suspend_timing()
854 					  suspend_clocksource->mask);  in clocksource_stop_suspend_timing()
855 		nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,  in clocksource_stop_suspend_timing()
856 				       suspend_clocksource->shift);  in clocksource_stop_suspend_timing()
860 	 * Disable the suspend timer to save power if current clocksource is  in clocksource_stop_suspend_timing()
861 	 * not the suspend timer.  in clocksource_stop_suspend_timing()
863 	if (!clocksource_is_suspend(cs) && suspend_clocksource->disable)  in clocksource_stop_suspend_timing()
864 		suspend_clocksource->disable(suspend_clocksource);  in clocksource_stop_suspend_timing()
870  * clocksource_suspend - suspend the clocksource(s)
877 		if (cs->suspend)  in clocksource_suspend()
878 			cs->suspend(cs);  in clocksource_suspend()
882  * clocksource_resume - resume the clocksource(s)
889 		if (cs->resume)  in clocksource_resume()
890 			cs->resume(cs);  in clocksource_resume()
896  * clocksource_touch_watchdog - Update watchdog
900  * was stopped in code which holds watchdog_lock.
908  * clocksource_max_adjustment- Returns max adjustment amount
918 	ret = (u64)cs->mult * 11;  in clocksource_max_adjustment()
924  * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
932  * NOTE: This function includes a safety margin of 50%, in other words, we
935  * delayed timers or bad hardware, which might result in time intervals that
944 	 * cyc2ns() function without overflowing a 64-bit result.  in clocks_calc_max_nsecs()
956 	max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);  in clocks_calc_max_nsecs()
969  * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
975 	cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,  in clocksource_update_max_deferment()
976 						cs->maxadj, cs->mask,  in clocksource_update_max_deferment()
977 						&cs->max_cycles);  in clocksource_update_max_deferment()
995 		if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))  in clocksource_find_best()
1019 		if (strcmp(cs->name, override_name) != 0)  in __clocksource_select()
1022 		 * Check to make sure we don't switch to a non-highres  in __clocksource_select()
1023 		 * capable clocksource if the tick code is in oneshot  in __clocksource_select()
1026 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {  in __clocksource_select()
1028 			if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in __clocksource_select()
1029 …pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/N…  in __clocksource_select()
1030 					cs->name);  in __clocksource_select()
1037 				pr_info("Override clocksource %s is not currently HRT compatible - deferring\n",  in __clocksource_select()
1038 					cs->name);  in __clocksource_select()
1048 		pr_info("Switched to clocksource %s\n", best->name);  in __clocksource_select()
1054  * clocksource_select - Select the best clocksource available
1072  * clocksource_done_booting - Called near the end of core bootup
1103 		if (tmp->rating < cs->rating)  in clocksource_enqueue()
1105 		entry = &tmp->list;  in clocksource_enqueue()
1107 	list_add(&cs->list, entry);  in clocksource_enqueue()
1111  * __clocksource_update_freq_scale - Used update clocksource with new freq
1116  * This should only be called from the clocksource->enable() method.
1127 	 * Default clocksources are *special* and self-define their mult/shift.  in __clocksource_update_freq_scale()
1133 		 * wrapping around. For clocksources which have a mask > 32-bit  in __clocksource_update_freq_scale()
1136 		 * amount. That results in a shift value of 24 for a  in __clocksource_update_freq_scale()
1137 		 * clocksource with mask >= 40-bit and f >= 4GHz. That maps to  in __clocksource_update_freq_scale()
1140 		sec = cs->mask;  in __clocksource_update_freq_scale()
1145 		else if (sec > 600 && cs->mask > UINT_MAX)  in __clocksource_update_freq_scale()
1148 		clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,  in __clocksource_update_freq_scale()
1154 	 * If both scale and freq are non-zero, calculate the clock  in __clocksource_update_freq_scale()
1157 	 * take the tens-of-milliseconds WATCHDOG_THRESHOLD value for the  in __clocksource_update_freq_scale()
1162 	if (scale && freq && !cs->uncertainty_margin) {  in __clocksource_update_freq_scale()
1163 		cs->uncertainty_margin = NSEC_PER_SEC / (scale * freq);  in __clocksource_update_freq_scale()
1164 		if (cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW)  in __clocksource_update_freq_scale()
1165 			cs->uncertainty_margin = 2 * WATCHDOG_MAX_SKEW;  in __clocksource_update_freq_scale()
1166 	} else if (!cs->uncertainty_margin) {  in __clocksource_update_freq_scale()
1167 		cs->uncertainty_margin = WATCHDOG_THRESHOLD;  in __clocksource_update_freq_scale()
1169 	WARN_ON_ONCE(cs->uncertainty_margin < 2 * WATCHDOG_MAX_SKEW);  in __clocksource_update_freq_scale()
1175 	cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
1176 	while (freq && ((cs->mult + cs->maxadj < cs->mult)  in __clocksource_update_freq_scale()
1177 		|| (cs->mult - cs->maxadj > cs->mult))) {  in __clocksource_update_freq_scale()
1178 		cs->mult >>= 1;  in __clocksource_update_freq_scale()
1179 		cs->shift--;  in __clocksource_update_freq_scale()
1180 		cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
1184 	 * Only warn for *special* clocksources that self-define  in __clocksource_update_freq_scale()
1187 	WARN_ONCE(cs->mult + cs->maxadj < cs->mult,  in __clocksource_update_freq_scale()
1189 		cs->name);  in __clocksource_update_freq_scale()
1194 		cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);  in __clocksource_update_freq_scale()
1199  * __clocksource_register_scale - Used to install new clocksources
1204  * Returns -EBUSY if registration fails, zero otherwise.
1215 	if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))  in __clocksource_register_scale()
1216 		cs->id = CSID_GENERIC;  in __clocksource_register_scale()
1217 	if (cs->vdso_clock_mode < 0 ||  in __clocksource_register_scale()
1218 	    cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) {  in __clocksource_register_scale()
1220 			cs->name, cs->vdso_clock_mode);  in __clocksource_register_scale()
1221 		cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE;  in __clocksource_register_scale()
1245 	list_del(&cs->list);  in __clocksource_change_rating()
1246 	cs->rating = rating;  in __clocksource_change_rating()
1251  * clocksource_change_rating - Change the rating of a registered clocksource
1282 			return -EBUSY;  in clocksource_unbind()
1289 			return -EBUSY;  in clocksource_unbind()
1294 		 * Select and try to install a replacement suspend clocksource.  in clocksource_unbind()
1295 		 * If no replacement suspend clocksource, we will just let the  in clocksource_unbind()
1296 		 * clocksource go and have no suspend clocksource.  in clocksource_unbind()
1303 	list_del_init(&cs->list);  in clocksource_unbind()
1310  * clocksource_unregister - remove a registered clocksource
1318 	if (!list_empty(&cs->list))  in clocksource_unregister()
1327  * current_clocksource_show - sysfs interface for current clocksource
1341 	count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);  in current_clocksource_show()
1353 		return -EINVAL;  in sysfs_get_uname()
1356 	if (buf[cnt-1] == '\n')  in sysfs_get_uname()
1357 		cnt--;  in sysfs_get_uname()
1365  * current_clocksource_store - interface for manually overriding clocksource
1393  * unbind_clocksource_store - interface for manually unbinding clocksource
1413 	ret = -ENODEV;  in unbind_clocksource_store()
1416 		if (strcmp(cs->name, name))  in unbind_clocksource_store()
1428  * available_clocksource_show - sysfs interface for listing clocksource
1445 		 * Don't show non-HRES clocksource if the tick code is  in available_clocksource_show()
1446 		 * in one shot mode (highres=on or nohz=on)  in available_clocksource_show()
1449 		    (src->flags & CLOCK_SOURCE_VALID_FOR_HRES))  in available_clocksource_show()
1451 				  max((ssize_t)PAGE_SIZE - count, (ssize_t)0),  in available_clocksource_show()
1452 				  "%s ", src->name);  in available_clocksource_show()
1457 			  max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n");  in available_clocksource_show()
1496  * boot_override_clocksource - boot clock override
1514  * boot_override_clock - Compatibility layer for deprecated boot option
1523 		pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n");  in boot_override_clock()
1526 	pr_warn("clock= boot option is deprecated - use clocksource=xyz\n");  in boot_override_clock()