Lines Matching +full:local +full:- +full:timer +full:- +full:stop
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
26 #include <linux/posix-timers.h>
32 #include "tick-internal.h"
34 #include <trace/events/timer.h>
37 * Per-CPU nohz control structure
63 * 64-bit can do a quick check without holding the jiffies lock and in tick_do_update_jiffies64()
67 * 32-bit cannot do that because the store of 'tick_next_period' in tick_do_update_jiffies64()
68 * consists of two 32-bit stores, and the first store could be in tick_do_update_jiffies64()
93 * Re-evaluate with the lock held. Another CPU might have done the in tick_do_update_jiffies64()
133 * A plain store is good enough on 32-bit, as the quick check in tick_do_update_jiffies64()
172 tick_next_period += TICK_NSEC - rem; in tick_init_jiffy_update()
217 if (ts->last_tick_jiffies != jiffies) { in tick_sched_do_timer()
218 ts->stalled_jiffies = 0; in tick_sched_do_timer()
219 ts->last_tick_jiffies = READ_ONCE(jiffies); in tick_sched_do_timer()
221 if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) { in tick_sched_do_timer()
223 ts->stalled_jiffies = 0; in tick_sched_do_timer()
224 ts->last_tick_jiffies = READ_ONCE(jiffies); in tick_sched_do_timer()
228 if (ts->inidle) in tick_sched_do_timer()
229 ts->got_idle_tick = 1; in tick_sched_do_timer()
243 if (ts->tick_stopped) { in tick_sched_handle()
246 ts->idle_jiffies++; in tick_sched_handle()
252 ts->next_tick = 0; in tick_sched_handle()
314 if (check_tick_dependency(&ts->tick_dep_mask)) in can_stop_full_tick()
317 if (check_tick_dependency(¤t->tick_dep_mask)) in can_stop_full_tick()
320 if (check_tick_dependency(¤t->signal->tick_dep_mask)) in can_stop_full_tick()
336 * re-evaluate its dependency on the tick and restart it if necessary.
350 * re-evaluate its dependency on the tick and restart it if necessary.
368 * activate_task() STORE p->tick_dep_mask in tick_nohz_kick_task()
369 * STORE p->on_rq in tick_nohz_kick_task()
371 * LOCK rq->lock LOAD p->on_rq in tick_nohz_kick_task()
374 * LOAD p->tick_dep_mask in tick_nohz_kick_task()
385 * STORE p->cpu = @cpu in tick_nohz_kick_task()
387 * LOCK rq->lock in tick_nohz_kick_task()
388 * smp_mb__after_spin_lock() STORE p->tick_dep_mask in tick_nohz_kick_task()
390 * LOAD p->tick_dep_mask LOAD p->cpu in tick_nohz_kick_task()
401 * Kick all full dynticks CPUs in order to force these to re-evaluate
442 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
443 * manage event-throttling.
452 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
455 /* Perf needs local kick that is NMI safe */ in tick_nohz_dep_set_cpu()
459 /* Remote IRQ work not NMI-safe */ in tick_nohz_dep_set_cpu()
472 atomic_andnot(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_clear_cpu()
477 * Set a per-task tick dependency. RCU needs this. Also posix CPU timers
482 if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) in tick_nohz_dep_set_task()
489 atomic_andnot(BIT(bit), &tsk->tick_dep_mask); in tick_nohz_dep_clear_task()
494 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
501 struct signal_struct *sig = tsk->signal; in tick_nohz_dep_set_signal()
503 prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask); in tick_nohz_dep_set_signal()
507 lockdep_assert_held(&tsk->sighand->siglock); in tick_nohz_dep_set_signal()
515 atomic_andnot(BIT(bit), &sig->tick_dep_mask); in tick_nohz_dep_clear_signal()
519 * Re-evaluate the need for the tick as we switch the current task.
532 if (ts->tick_stopped) { in __tick_nohz_task_switch()
533 if (atomic_read(¤t->tick_dep_mask) || in __tick_nohz_task_switch()
534 atomic_read(¤t->signal->tick_dep_mask)) in __tick_nohz_task_switch()
539 /* Get the boot-time nohz CPU list from the kernel parameters. */
561 return tick_nohz_cpu_hotpluggable(cpu) ? 0 : -EBUSY; in tick_nohz_cpu_down()
577 pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support IRQ work self-IPIs\n"); in tick_nohz_init()
607 * NOHZ - aka dynamic tick functionality
629 return ts->tick_stopped; in tick_nohz_tick_stopped()
636 return ts->tick_stopped; in tick_nohz_tick_stopped_cpu()
640 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
666 if (WARN_ON_ONCE(!ts->idle_active)) in tick_nohz_stop_idle()
669 delta = ktime_sub(now, ts->idle_entrytime); in tick_nohz_stop_idle()
671 write_seqcount_begin(&ts->idle_sleeptime_seq); in tick_nohz_stop_idle()
673 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in tick_nohz_stop_idle()
675 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in tick_nohz_stop_idle()
677 ts->idle_entrytime = now; in tick_nohz_stop_idle()
678 ts->idle_active = 0; in tick_nohz_stop_idle()
679 write_seqcount_end(&ts->idle_sleeptime_seq); in tick_nohz_stop_idle()
686 write_seqcount_begin(&ts->idle_sleeptime_seq); in tick_nohz_start_idle()
687 ts->idle_entrytime = ktime_get(); in tick_nohz_start_idle()
688 ts->idle_active = 1; in tick_nohz_start_idle()
689 write_seqcount_end(&ts->idle_sleeptime_seq); in tick_nohz_start_idle()
701 return -1; in get_cpu_sleep_time_us()
708 seq = read_seqcount_begin(&ts->idle_sleeptime_seq); in get_cpu_sleep_time_us()
710 if (ts->idle_active && compute_delta) { in get_cpu_sleep_time_us()
711 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_sleep_time_us()
717 } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq)); in get_cpu_sleep_time_us()
724 * get_cpu_idle_time_us - get the total idle time of a CPU
738 * This function returns -1 if NOHZ is not enabled.
744 return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime, in get_cpu_idle_time_us()
750 * get_cpu_iowait_time_us - get the total iowait time of a CPU
764 * This function returns -1 if NOHZ is not enabled.
770 return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime, in get_cpu_iowait_time_us()
777 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
778 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
781 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_restart()
783 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
784 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
787 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_restart()
791 * Reset to make sure the next tick stop doesn't get fooled by past in tick_nohz_restart()
794 ts->next_tick = 0; in tick_nohz_restart()
814 ts->last_jiffies = basejiff; in tick_nohz_next_event()
815 ts->timer_expires_base = basemono; in tick_nohz_next_event()
820 * Aside of that, check whether the local timer softirq is in tick_nohz_next_event()
822 * because there is an already expired timer, so it will request in tick_nohz_next_event()
823 * immediate expiry, which rearms the hardware timer with a in tick_nohz_next_event()
832 * Get the next pending timer. If high resolution in tick_nohz_next_event()
833 * timers are enabled this only takes the timer wheel in tick_nohz_next_event()
839 ts->next_timer = next_tick; in tick_nohz_next_event()
848 * force prod the timer. in tick_nohz_next_event()
850 delta = next_tick - basemono; in tick_nohz_next_event()
853 * Tell the timer code that the base is not idle, i.e. undo in tick_nohz_next_event()
858 * We've not stopped the tick yet, and there's a timer in the in tick_nohz_next_event()
861 if (!ts->tick_stopped) { in tick_nohz_next_event()
862 ts->timer_expires = 0; in tick_nohz_next_event()
874 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) in tick_nohz_next_event()
878 if (delta < (KTIME_MAX - basemono)) in tick_nohz_next_event()
883 ts->timer_expires = min_t(u64, expires, next_tick); in tick_nohz_next_event()
886 return ts->timer_expires; in tick_nohz_next_event()
892 u64 basemono = ts->timer_expires_base; in tick_nohz_stop_tick()
893 u64 expires = ts->timer_expires; in tick_nohz_stop_tick()
895 /* Make sure we won't be trying to stop it twice in a row. */ in tick_nohz_stop_tick()
896 ts->timer_expires_base = 0; in tick_nohz_stop_tick()
901 * the tick timer next, which might be this CPU as well. If we in tick_nohz_stop_tick()
908 ts->do_timer_last = 1; in tick_nohz_stop_tick()
910 ts->do_timer_last = 0; in tick_nohz_stop_tick()
914 if (ts->tick_stopped && (expires == ts->next_tick)) { in tick_nohz_stop_tick()
916 if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) in tick_nohz_stop_tick()
920 …printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->exp… in tick_nohz_stop_tick()
921 basemono, ts->next_tick, dev->next_event, in tick_nohz_stop_tick()
922 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); in tick_nohz_stop_tick()
932 if (!ts->tick_stopped) { in tick_nohz_stop_tick()
936 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_tick()
937 ts->tick_stopped = 1; in tick_nohz_stop_tick()
941 ts->next_tick = expires; in tick_nohz_stop_tick()
944 * If the expiration time == KTIME_MAX, then we simply stop in tick_nohz_stop_tick()
945 * the tick timer. in tick_nohz_stop_tick()
948 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_tick()
949 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_tick()
955 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_tick()
956 hrtimer_start(&ts->sched_timer, expires, in tick_nohz_stop_tick()
959 hrtimer_set_expires(&ts->sched_timer, expires); in tick_nohz_stop_tick()
966 ts->timer_expires_base = 0; in tick_nohz_retain_tick()
985 * Clear the timer idle flag, so we avoid IPIs on remote queueing and in tick_nohz_restart_sched_tick()
993 /* Cancel the scheduled timer and restore the tick: */ in tick_nohz_restart_sched_tick()
994 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
1006 else if (ts->tick_stopped) in __tick_nohz_full_update_tick()
1016 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_update_tick()
1054 pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n", in report_idle_softirq()
1066 * the CPU which runs the tick timer next. If we don't drop in can_stop_idle_tick()
1077 ts->next_tick = 0; in can_stop_idle_tick()
1081 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
1098 /* Should not happen for nohz-full */ in can_stop_idle_tick()
1107 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
1109 * When the next event is more than a tick into the future, stop the idle tick
1119 * tick timer expiration time is known already. in tick_nohz_idle_stop_tick()
1121 if (ts->timer_expires_base) in tick_nohz_idle_stop_tick()
1122 expires = ts->timer_expires; in tick_nohz_idle_stop_tick()
1128 ts->idle_calls++; in tick_nohz_idle_stop_tick()
1131 int was_stopped = ts->tick_stopped; in tick_nohz_idle_stop_tick()
1135 ts->idle_sleeps++; in tick_nohz_idle_stop_tick()
1136 ts->idle_expires = expires; in tick_nohz_idle_stop_tick()
1138 if (!was_stopped && ts->tick_stopped) { in tick_nohz_idle_stop_tick()
1139 ts->idle_jiffies = ts->last_jiffies; in tick_nohz_idle_stop_tick()
1158 * tick_nohz_idle_enter - prepare for entering idle on the current CPU
1172 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_enter()
1174 ts->inidle = 1; in tick_nohz_idle_enter()
1181 * tick_nohz_irq_exit - Notify the tick about IRQ exit
1183 * A timer may have been added/modified/deleted either by the current IRQ,
1186 * re-evaluation of the next tick. Depending on the context:
1189 * time accounting. The next tick will be re-evaluated on the next idle
1196 * 2.2) If there is no tick dependency, (re-)evaluate the next tick and
1197 * stop/update it accordingly.
1203 if (ts->inidle) in tick_nohz_irq_exit()
1210 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
1216 if (ts->got_idle_tick) { in tick_nohz_idle_got_tick()
1217 ts->got_idle_tick = 0; in tick_nohz_idle_got_tick()
1224 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
1232 return __this_cpu_read(tick_cpu_device.evtdev)->next_event; in tick_nohz_get_next_hrtimer()
1236 * tick_nohz_get_sleep_length - return the expected length of the current sleep
1254 ktime_t now = ts->idle_entrytime; in tick_nohz_get_sleep_length()
1257 WARN_ON_ONCE(!ts->inidle); in tick_nohz_get_sleep_length()
1259 *delta_next = ktime_sub(dev->next_event, now); in tick_nohz_get_sleep_length()
1269 * If the next highres timer to expire is earlier than 'next_event', the in tick_nohz_get_sleep_length()
1273 hrtimer_next_event_without(&ts->sched_timer)); in tick_nohz_get_sleep_length()
1279 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
1288 return ts->idle_calls; in tick_nohz_get_idle_calls_cpu()
1292 * tick_nohz_get_idle_calls - return the current idle calls counter value
1300 return ts->idle_calls; in tick_nohz_get_idle_calls()
1308 ts->idle_exittime = now; in tick_nohz_account_idle_time()
1317 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_time()
1329 if (ts->tick_stopped) { in tick_nohz_idle_restart_tick()
1347 * tick_nohz_idle_exit - Update the tick upon idle task exit
1357 * then re-evaluate the next tick and try to keep it stopped
1370 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
1371 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_exit()
1373 ts->inidle = 0; in tick_nohz_idle_exit()
1374 idle_active = ts->idle_active; in tick_nohz_idle_exit()
1375 tick_stopped = ts->tick_stopped; in tick_nohz_idle_exit()
1390 * In low-resolution mode, the tick handler must be implemented directly
1393 * low-resolution mode (see hrtimer_run_queues()).
1395 * This low-resolution handler still makes use of some hrtimer APIs meanwhile
1404 dev->next_event = KTIME_MAX; in tick_nohz_lowres_handler()
1411 * - to the idle task if in dynticks-idle in tick_nohz_lowres_handler()
1412 * - to IRQ exit if in full-dynticks. in tick_nohz_lowres_handler()
1414 if (likely(!ts->tick_stopped)) { in tick_nohz_lowres_handler()
1415 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_nohz_lowres_handler()
1416 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_lowres_handler()
1425 ts->nohz_mode = mode; in tick_nohz_activate()
1432 * tick_nohz_switch_to_nohz - switch to NOHZ mode
1449 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_nohz_switch_to_nohz()
1453 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1454 hrtimer_forward_now(&ts->sched_timer, TICK_NSEC); in tick_nohz_switch_to_nohz()
1455 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_switch_to_nohz()
1464 if (!ts->idle_active && !ts->tick_stopped) in tick_nohz_irq_enter()
1467 if (ts->idle_active) in tick_nohz_irq_enter()
1473 * rare case (typically stop machine). So we must make sure we have a in tick_nohz_irq_enter()
1476 if (ts->tick_stopped) in tick_nohz_irq_enter()
1498 * High resolution timer specific code
1502 * We rearm the timer until we get disabled by the idle code.
1505 static enum hrtimer_restart tick_nohz_highres_handler(struct hrtimer *timer) in tick_nohz_highres_handler() argument
1508 container_of(timer, struct tick_sched, sched_timer); in tick_nohz_highres_handler()
1521 ts->next_tick = 0; in tick_nohz_highres_handler()
1525 * - to the idle task if in dynticks-idle in tick_nohz_highres_handler()
1526 * - to IRQ exit if in full-dynticks. in tick_nohz_highres_handler()
1528 if (unlikely(ts->tick_stopped)) in tick_nohz_highres_handler()
1531 hrtimer_forward(timer, now, TICK_NSEC); in tick_nohz_highres_handler()
1547 * tick_setup_sched_timer - setup the tick emulation timer
1554 /* Emulate tick processing via per-CPU hrtimers: */ in tick_setup_sched_timer()
1555 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_setup_sched_timer()
1556 ts->sched_timer.function = tick_nohz_highres_handler; in tick_setup_sched_timer()
1558 /* Get the next period (per-CPU) */ in tick_setup_sched_timer()
1559 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1566 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1569 hrtimer_forward(&ts->sched_timer, now, TICK_NSEC); in tick_setup_sched_timer()
1570 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); in tick_setup_sched_timer()
1583 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1584 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1587 idle_sleeptime = ts->idle_sleeptime; in tick_cancel_sched_timer()
1588 iowait_sleeptime = ts->iowait_sleeptime; in tick_cancel_sched_timer()
1589 idle_calls = ts->idle_calls; in tick_cancel_sched_timer()
1590 idle_sleeps = ts->idle_sleeps; in tick_cancel_sched_timer()
1592 ts->idle_sleeptime = idle_sleeptime; in tick_cancel_sched_timer()
1593 ts->iowait_sleeptime = iowait_sleeptime; in tick_cancel_sched_timer()
1594 ts->idle_calls = idle_calls; in tick_cancel_sched_timer()
1595 ts->idle_sleeps = idle_sleeps; in tick_cancel_sched_timer()
1617 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1623 * Called cyclically from the hrtimer softirq (driven by the timer
1624 * softirq). 'allow_nohz' signals that we can switch into low-res NOHZ
1632 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1635 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()