186 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)  in tick_sched_do_timer()  argument
217 	if (ts->last_tick_jiffies != jiffies) {  in tick_sched_do_timer()
218 		ts->stalled_jiffies = 0;  in tick_sched_do_timer()
219 		ts->last_tick_jiffies = READ_ONCE(jiffies);  in tick_sched_do_timer()
221 		if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {  in tick_sched_do_timer()
223 			ts->stalled_jiffies = 0;  in tick_sched_do_timer()
224 			ts->last_tick_jiffies = READ_ONCE(jiffies);  in tick_sched_do_timer()
228 	if (ts->inidle)  in tick_sched_do_timer()
229 		ts->got_idle_tick = 1;  in tick_sched_do_timer()
232 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)  in tick_sched_handle()  argument
243 	if (ts->tick_stopped) {  in tick_sched_handle()
246 			ts->idle_jiffies++;  in tick_sched_handle()
252 		ts->next_tick = 0;  in tick_sched_handle()
304 static bool can_stop_full_tick(int cpu, struct tick_sched *ts)  in can_stop_full_tick()  argument
314 	if (check_tick_dependency(&ts->tick_dep_mask))  in can_stop_full_tick()
448 	struct tick_sched *ts;  in tick_nohz_dep_set_cpu()  local
450 	ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_set_cpu()
452 	prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_set_cpu()
470 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_dep_clear_cpu()  local
472 	atomic_andnot(BIT(bit), &ts->tick_dep_mask);  in tick_nohz_dep_clear_cpu()
525 	struct tick_sched *ts;  in __tick_nohz_task_switch()  local
530 	ts = this_cpu_ptr(&tick_cpu_sched);  in __tick_nohz_task_switch()
532 	if (ts->tick_stopped) {  in __tick_nohz_task_switch()
627 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_tick_stopped()  local
629 	return ts->tick_stopped;  in tick_nohz_tick_stopped()
634 	struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);  in tick_nohz_tick_stopped_cpu()  local
636 	return ts->tick_stopped;  in tick_nohz_tick_stopped_cpu()
662 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
666 	if (WARN_ON_ONCE(!ts->idle_active))  in tick_nohz_stop_idle()
669 	delta = ktime_sub(now, ts->idle_entrytime);  in tick_nohz_stop_idle()
671 	write_seqcount_begin(&ts->idle_sleeptime_seq);  in tick_nohz_stop_idle()
673 		ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);  in tick_nohz_stop_idle()
675 		ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in tick_nohz_stop_idle()
677 	ts->idle_entrytime = now;  in tick_nohz_stop_idle()
678 	ts->idle_active = 0;  in tick_nohz_stop_idle()
679 	write_seqcount_end(&ts->idle_sleeptime_seq);  in tick_nohz_stop_idle()
684 static void tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
686 	write_seqcount_begin(&ts->idle_sleeptime_seq);  in tick_nohz_start_idle()
687 	ts->idle_entrytime = ktime_get();  in tick_nohz_start_idle()
688 	ts->idle_active = 1;  in tick_nohz_start_idle()
689 	write_seqcount_end(&ts->idle_sleeptime_seq);  in tick_nohz_start_idle()
694 static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,  in get_cpu_sleep_time_us()  argument
708 		seq = read_seqcount_begin(&ts->idle_sleeptime_seq);  in get_cpu_sleep_time_us()
710 		if (ts->idle_active && compute_delta) {  in get_cpu_sleep_time_us()
711 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_sleep_time_us()
717 	} while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));  in get_cpu_sleep_time_us()
742 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
744 	return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,  in get_cpu_idle_time_us()
768 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_iowait_time_us()  local
770 	return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,  in get_cpu_iowait_time_us()
775 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
777 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
778 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);  in tick_nohz_restart()
781 	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);  in tick_nohz_restart()
783 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_restart()
784 		hrtimer_start_expires(&ts->sched_timer,  in tick_nohz_restart()
787 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_restart()
794 	ts->next_tick = 0;  in tick_nohz_restart()
802 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)  in tick_nohz_next_event()  argument
814 	ts->last_jiffies = basejiff;  in tick_nohz_next_event()
815 	ts->timer_expires_base = basemono;  in tick_nohz_next_event()
839 		ts->next_timer = next_tick;  in tick_nohz_next_event()
861 		if (!ts->tick_stopped) {  in tick_nohz_next_event()
862 			ts->timer_expires = 0;  in tick_nohz_next_event()
874 	    (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))  in tick_nohz_next_event()
883 	ts->timer_expires = min_t(u64, expires, next_tick);  in tick_nohz_next_event()
886 	return ts->timer_expires;  in tick_nohz_next_event()
889 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)  in tick_nohz_stop_tick()  argument
892 	u64 basemono = ts->timer_expires_base;  in tick_nohz_stop_tick()
893 	u64 expires = ts->timer_expires;  in tick_nohz_stop_tick()
896 	ts->timer_expires_base = 0;  in tick_nohz_stop_tick()
908 		ts->do_timer_last = 1;  in tick_nohz_stop_tick()
910 		ts->do_timer_last = 0;  in tick_nohz_stop_tick()
914 	if (ts->tick_stopped && (expires == ts->next_tick)) {  in tick_nohz_stop_tick()
916 		if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))  in tick_nohz_stop_tick()
920 …printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->exp…  in tick_nohz_stop_tick()
921 			    basemono, ts->next_tick, dev->next_event,  in tick_nohz_stop_tick()
922 			    hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));  in tick_nohz_stop_tick()
932 	if (!ts->tick_stopped) {  in tick_nohz_stop_tick()
936 		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_tick()
937 		ts->tick_stopped = 1;  in tick_nohz_stop_tick()
941 	ts->next_tick = expires;  in tick_nohz_stop_tick()
948 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_tick()
949 			hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_tick()
955 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {  in tick_nohz_stop_tick()
956 		hrtimer_start(&ts->sched_timer, expires,  in tick_nohz_stop_tick()
959 		hrtimer_set_expires(&ts->sched_timer, expires);  in tick_nohz_stop_tick()
964 static void tick_nohz_retain_tick(struct tick_sched *ts)  in tick_nohz_retain_tick()  argument
966 	ts->timer_expires_base = 0;  in tick_nohz_retain_tick()
970 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)  in tick_nohz_stop_sched_tick()  argument
972 	if (tick_nohz_next_event(ts, cpu))  in tick_nohz_stop_sched_tick()
973 		tick_nohz_stop_tick(ts, cpu);  in tick_nohz_stop_sched_tick()
975 		tick_nohz_retain_tick(ts);  in tick_nohz_stop_sched_tick()
979 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
994 	ts->tick_stopped  = 0;  in tick_nohz_restart_sched_tick()
995 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
998 static void __tick_nohz_full_update_tick(struct tick_sched *ts,  in __tick_nohz_full_update_tick()  argument
1004 	if (can_stop_full_tick(cpu, ts))  in __tick_nohz_full_update_tick()
1005 		tick_nohz_stop_sched_tick(ts, cpu);  in __tick_nohz_full_update_tick()
1006 	else if (ts->tick_stopped)  in __tick_nohz_full_update_tick()
1007 		tick_nohz_restart_sched_tick(ts, now);  in __tick_nohz_full_update_tick()
1011 static void tick_nohz_full_update_tick(struct tick_sched *ts)  in tick_nohz_full_update_tick()  argument
1016 	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)  in tick_nohz_full_update_tick()
1019 	__tick_nohz_full_update_tick(ts, ktime_get());  in tick_nohz_full_update_tick()
1061 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)  in can_stop_idle_tick()  argument
1077 		ts->next_tick = 0;  in can_stop_idle_tick()
1081 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))  in can_stop_idle_tick()
1113 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_stop_tick()  local
1121 	if (ts->timer_expires_base)  in tick_nohz_idle_stop_tick()
1122 		expires = ts->timer_expires;  in tick_nohz_idle_stop_tick()
1123 	else if (can_stop_idle_tick(cpu, ts))  in tick_nohz_idle_stop_tick()
1124 		expires = tick_nohz_next_event(ts, cpu);  in tick_nohz_idle_stop_tick()
1128 	ts->idle_calls++;  in tick_nohz_idle_stop_tick()
1131 		int was_stopped = ts->tick_stopped;  in tick_nohz_idle_stop_tick()
1133 		tick_nohz_stop_tick(ts, cpu);  in tick_nohz_idle_stop_tick()
1135 		ts->idle_sleeps++;  in tick_nohz_idle_stop_tick()
1136 		ts->idle_expires = expires;  in tick_nohz_idle_stop_tick()
1138 		if (!was_stopped && ts->tick_stopped) {  in tick_nohz_idle_stop_tick()
1139 			ts->idle_jiffies = ts->last_jiffies;  in tick_nohz_idle_stop_tick()
1143 		tick_nohz_retain_tick(ts);  in tick_nohz_idle_stop_tick()
1164 	struct tick_sched *ts;  in tick_nohz_idle_enter()  local
1170 	ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_enter()
1172 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_enter()
1174 	ts->inidle = 1;  in tick_nohz_idle_enter()
1175 	tick_nohz_start_idle(ts);  in tick_nohz_idle_enter()
1201 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_exit()  local
1203 	if (ts->inidle)  in tick_nohz_irq_exit()
1204 		tick_nohz_start_idle(ts);  in tick_nohz_irq_exit()
1206 		tick_nohz_full_update_tick(ts);  in tick_nohz_irq_exit()
1214 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_got_tick()  local
1216 	if (ts->got_idle_tick) {  in tick_nohz_idle_got_tick()
1217 		ts->got_idle_tick = 0;  in tick_nohz_idle_got_tick()
1248 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
1254 	ktime_t now = ts->idle_entrytime;  in tick_nohz_get_sleep_length()
1257 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_get_sleep_length()
1261 	if (!can_stop_idle_tick(cpu, ts))  in tick_nohz_get_sleep_length()
1264 	next_event = tick_nohz_next_event(ts, cpu);  in tick_nohz_get_sleep_length()
1273 			   hrtimer_next_event_without(&ts->sched_timer));  in tick_nohz_get_sleep_length()
1286 	struct tick_sched *ts = tick_get_tick_sched(cpu);  in tick_nohz_get_idle_calls_cpu()  local
1288 	return ts->idle_calls;  in tick_nohz_get_idle_calls_cpu()
1298 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_idle_calls()  local
1300 	return ts->idle_calls;  in tick_nohz_get_idle_calls()
1303 static void tick_nohz_account_idle_time(struct tick_sched *ts,  in tick_nohz_account_idle_time()  argument
1308 	ts->idle_exittime = now;  in tick_nohz_account_idle_time()
1317 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_account_idle_time()
1327 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_restart_tick()  local
1329 	if (ts->tick_stopped) {  in tick_nohz_idle_restart_tick()
1331 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_restart_tick()
1332 		tick_nohz_account_idle_time(ts, now);  in tick_nohz_idle_restart_tick()
1336 static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_idle_update_tick()  argument
1339 		__tick_nohz_full_update_tick(ts, now);  in tick_nohz_idle_update_tick()
1341 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_update_tick()
1343 	tick_nohz_account_idle_time(ts, now);  in tick_nohz_idle_update_tick()
1364 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_exit()  local
1370 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_idle_exit()
1371 	WARN_ON_ONCE(ts->timer_expires_base);  in tick_nohz_idle_exit()
1373 	ts->inidle = 0;  in tick_nohz_idle_exit()
1374 	idle_active = ts->idle_active;  in tick_nohz_idle_exit()
1375 	tick_stopped = ts->tick_stopped;  in tick_nohz_idle_exit()
1381 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
1384 		tick_nohz_idle_update_tick(ts, now);  in tick_nohz_idle_exit()
1400 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_lowres_handler()  local
1406 	tick_sched_do_timer(ts, now);  in tick_nohz_lowres_handler()
1407 	tick_sched_handle(ts, regs);  in tick_nohz_lowres_handler()
1414 	if (likely(!ts->tick_stopped)) {  in tick_nohz_lowres_handler()
1415 		hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);  in tick_nohz_lowres_handler()
1416 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_lowres_handler()
1421 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)  in tick_nohz_activate()  argument
1425 	ts->nohz_mode = mode;  in tick_nohz_activate()
1436 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_switch_to_nohz()  local
1446 	 * Recycle the hrtimer in 'ts', so we can share the  in tick_nohz_switch_to_nohz()
1449 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);  in tick_nohz_switch_to_nohz()
1453 	hrtimer_set_expires(&ts->sched_timer, next);  in tick_nohz_switch_to_nohz()
1454 	hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);  in tick_nohz_switch_to_nohz()
1455 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_switch_to_nohz()
1456 	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);  in tick_nohz_switch_to_nohz()
1461 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_enter()  local
1464 	if (!ts->idle_active && !ts->tick_stopped)  in tick_nohz_irq_enter()
1467 	if (ts->idle_active)  in tick_nohz_irq_enter()
1468 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1476 	if (ts->tick_stopped)  in tick_nohz_irq_enter()
1484 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }  in tick_nohz_activate()  argument
1507 	struct tick_sched *ts =  in tick_nohz_highres_handler()  local
1512 	tick_sched_do_timer(ts, now);  in tick_nohz_highres_handler()
1519 		tick_sched_handle(ts, regs);  in tick_nohz_highres_handler()
1521 		ts->next_tick = 0;  in tick_nohz_highres_handler()
1528 	if (unlikely(ts->tick_stopped))  in tick_nohz_highres_handler()
1551 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_setup_sched_timer()  local
1555 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);  in tick_setup_sched_timer()
1556 	ts->sched_timer.function = tick_nohz_highres_handler;  in tick_setup_sched_timer()
1559 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
1566 		hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
1569 	hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);  in tick_setup_sched_timer()
1570 	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);  in tick_setup_sched_timer()
1571 	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);  in tick_setup_sched_timer()
1578 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_cancel_sched_timer()  local
1583 	if (ts->sched_timer.base)  in tick_cancel_sched_timer()
1584 		hrtimer_cancel(&ts->sched_timer);  in tick_cancel_sched_timer()
1587 	idle_sleeptime = ts->idle_sleeptime;  in tick_cancel_sched_timer()
1588 	iowait_sleeptime = ts->iowait_sleeptime;  in tick_cancel_sched_timer()
1589 	idle_calls = ts->idle_calls;  in tick_cancel_sched_timer()
1590 	idle_sleeps = ts->idle_sleeps;  in tick_cancel_sched_timer()
1591 	memset(ts, 0, sizeof(*ts));  in tick_cancel_sched_timer()
1592 	ts->idle_sleeptime = idle_sleeptime;  in tick_cancel_sched_timer()
1593 	ts->iowait_sleeptime = iowait_sleeptime;  in tick_cancel_sched_timer()
1594 	ts->idle_calls = idle_calls;  in tick_cancel_sched_timer()
1595 	ts->idle_sleeps = idle_sleeps;  in tick_cancel_sched_timer()
1615 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_oneshot_notify()  local
1617 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
1630 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_check_oneshot_change()  local
1632 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
1635 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)  in tick_check_oneshot_change()