Lines Matching full:ts
122 static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) in tick_sched_do_timer() argument
149 if (ts->inidle) in tick_sched_do_timer()
150 ts->got_idle_tick = 1; in tick_sched_do_timer()
153 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) in tick_sched_handle() argument
164 if (ts->tick_stopped) { in tick_sched_handle()
167 ts->idle_jiffies++; in tick_sched_handle()
173 ts->next_tick = 0; in tick_sched_handle()
219 static bool can_stop_full_tick(int cpu, struct tick_sched *ts) in can_stop_full_tick() argument
229 if (check_tick_dependency(&ts->tick_dep_mask)) in can_stop_full_tick()
325 struct tick_sched *ts; in tick_nohz_dep_set_cpu() local
327 ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_set_cpu()
329 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_set_cpu()
347 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_dep_clear_cpu() local
349 atomic_andnot(BIT(bit), &ts->tick_dep_mask); in tick_nohz_dep_clear_cpu()
403 struct tick_sched *ts; in __tick_nohz_task_switch() local
410 ts = this_cpu_ptr(&tick_cpu_sched); in __tick_nohz_task_switch()
412 if (ts->tick_stopped) { in __tick_nohz_task_switch()
505 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_tick_stopped() local
507 return ts->tick_stopped; in tick_nohz_tick_stopped()
512 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu); in tick_nohz_tick_stopped_cpu() local
514 return ts->tick_stopped; in tick_nohz_tick_stopped_cpu()
544 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time) in update_ts_time_stats() argument
548 if (ts->idle_active) { in update_ts_time_stats()
549 delta = ktime_sub(now, ts->idle_entrytime); in update_ts_time_stats()
551 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta); in update_ts_time_stats()
553 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); in update_ts_time_stats()
554 ts->idle_entrytime = now; in update_ts_time_stats()
562 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now) in tick_nohz_stop_idle() argument
564 update_ts_time_stats(smp_processor_id(), ts, now, NULL); in tick_nohz_stop_idle()
565 ts->idle_active = 0; in tick_nohz_stop_idle()
570 static void tick_nohz_start_idle(struct tick_sched *ts) in tick_nohz_start_idle() argument
572 ts->idle_entrytime = ktime_get(); in tick_nohz_start_idle()
573 ts->idle_active = 1; in tick_nohz_start_idle()
593 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_idle_time_us() local
601 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_idle_time_us()
602 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
604 if (ts->idle_active && !nr_iowait_cpu(cpu)) { in get_cpu_idle_time_us()
605 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_idle_time_us()
607 idle = ktime_add(ts->idle_sleeptime, delta); in get_cpu_idle_time_us()
609 idle = ts->idle_sleeptime; in get_cpu_idle_time_us()
634 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in get_cpu_iowait_time_us() local
642 update_ts_time_stats(cpu, ts, now, last_update_time); in get_cpu_iowait_time_us()
643 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
645 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) { in get_cpu_iowait_time_us()
646 ktime_t delta = ktime_sub(now, ts->idle_entrytime); in get_cpu_iowait_time_us()
648 iowait = ktime_add(ts->iowait_sleeptime, delta); in get_cpu_iowait_time_us()
650 iowait = ts->iowait_sleeptime; in get_cpu_iowait_time_us()
658 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) in tick_nohz_restart() argument
660 hrtimer_cancel(&ts->sched_timer); in tick_nohz_restart()
661 hrtimer_set_expires(&ts->sched_timer, ts->last_tick); in tick_nohz_restart()
664 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_nohz_restart()
666 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_restart()
667 hrtimer_start_expires(&ts->sched_timer, in tick_nohz_restart()
670 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_restart()
677 ts->next_tick = 0; in tick_nohz_restart()
685 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) in tick_nohz_next_event() argument
697 ts->last_jiffies = basejiff; in tick_nohz_next_event()
698 ts->timer_expires_base = basemono; in tick_nohz_next_event()
722 ts->next_timer = next_tmr; in tick_nohz_next_event()
742 if (!ts->tick_stopped) { in tick_nohz_next_event()
743 ts->timer_expires = 0; in tick_nohz_next_event()
755 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last)) in tick_nohz_next_event()
764 ts->timer_expires = min_t(u64, expires, next_tick); in tick_nohz_next_event()
767 return ts->timer_expires; in tick_nohz_next_event()
770 static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_tick() argument
773 u64 basemono = ts->timer_expires_base; in tick_nohz_stop_tick()
774 u64 expires = ts->timer_expires; in tick_nohz_stop_tick()
778 ts->timer_expires_base = 0; in tick_nohz_stop_tick()
790 ts->do_timer_last = 1; in tick_nohz_stop_tick()
792 ts->do_timer_last = 0; in tick_nohz_stop_tick()
796 if (ts->tick_stopped && (expires == ts->next_tick)) { in tick_nohz_stop_tick()
798 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer)) in tick_nohz_stop_tick()
802 …printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->exp… in tick_nohz_stop_tick()
803 basemono, ts->next_tick, dev->next_event, in tick_nohz_stop_tick()
804 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer)); in tick_nohz_stop_tick()
814 if (!ts->tick_stopped) { in tick_nohz_stop_tick()
818 ts->last_tick = hrtimer_get_expires(&ts->sched_timer); in tick_nohz_stop_tick()
819 ts->tick_stopped = 1; in tick_nohz_stop_tick()
823 ts->next_tick = tick; in tick_nohz_stop_tick()
830 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) in tick_nohz_stop_tick()
831 hrtimer_cancel(&ts->sched_timer); in tick_nohz_stop_tick()
835 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { in tick_nohz_stop_tick()
836 hrtimer_start(&ts->sched_timer, tick, in tick_nohz_stop_tick()
839 hrtimer_set_expires(&ts->sched_timer, tick); in tick_nohz_stop_tick()
844 static void tick_nohz_retain_tick(struct tick_sched *ts) in tick_nohz_retain_tick() argument
846 ts->timer_expires_base = 0; in tick_nohz_retain_tick()
850 static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu) in tick_nohz_stop_sched_tick() argument
852 if (tick_nohz_next_event(ts, cpu)) in tick_nohz_stop_sched_tick()
853 tick_nohz_stop_tick(ts, cpu); in tick_nohz_stop_sched_tick()
855 tick_nohz_retain_tick(ts); in tick_nohz_stop_sched_tick()
859 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) in tick_nohz_restart_sched_tick() argument
875 ts->tick_stopped = 0; in tick_nohz_restart_sched_tick()
876 ts->idle_exittime = now; in tick_nohz_restart_sched_tick()
878 tick_nohz_restart(ts, now); in tick_nohz_restart_sched_tick()
881 static void tick_nohz_full_update_tick(struct tick_sched *ts) in tick_nohz_full_update_tick() argument
889 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) in tick_nohz_full_update_tick()
892 if (can_stop_full_tick(cpu, ts)) in tick_nohz_full_update_tick()
893 tick_nohz_stop_sched_tick(ts, cpu); in tick_nohz_full_update_tick()
894 else if (ts->tick_stopped) in tick_nohz_full_update_tick()
895 tick_nohz_restart_sched_tick(ts, ktime_get()); in tick_nohz_full_update_tick()
899 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) in can_stop_idle_tick() argument
915 ts->next_tick = 0; in can_stop_idle_tick()
919 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) in can_stop_idle_tick()
960 static void __tick_nohz_idle_stop_tick(struct tick_sched *ts) in __tick_nohz_idle_stop_tick() argument
969 if (ts->timer_expires_base) in __tick_nohz_idle_stop_tick()
970 expires = ts->timer_expires; in __tick_nohz_idle_stop_tick()
971 else if (can_stop_idle_tick(cpu, ts)) in __tick_nohz_idle_stop_tick()
972 expires = tick_nohz_next_event(ts, cpu); in __tick_nohz_idle_stop_tick()
976 ts->idle_calls++; in __tick_nohz_idle_stop_tick()
979 int was_stopped = ts->tick_stopped; in __tick_nohz_idle_stop_tick()
981 tick_nohz_stop_tick(ts, cpu); in __tick_nohz_idle_stop_tick()
983 ts->idle_sleeps++; in __tick_nohz_idle_stop_tick()
984 ts->idle_expires = expires; in __tick_nohz_idle_stop_tick()
986 if (!was_stopped && ts->tick_stopped) { in __tick_nohz_idle_stop_tick()
987 ts->idle_jiffies = ts->last_jiffies; in __tick_nohz_idle_stop_tick()
991 tick_nohz_retain_tick(ts); in __tick_nohz_idle_stop_tick()
1022 struct tick_sched *ts; in tick_nohz_idle_enter() local
1028 ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_enter()
1030 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_enter()
1032 ts->inidle = 1; in tick_nohz_idle_enter()
1033 tick_nohz_start_idle(ts); in tick_nohz_idle_enter()
1048 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_exit() local
1050 if (ts->inidle) in tick_nohz_irq_exit()
1051 tick_nohz_start_idle(ts); in tick_nohz_irq_exit()
1053 tick_nohz_full_update_tick(ts); in tick_nohz_irq_exit()
1061 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_got_tick() local
1063 if (ts->got_idle_tick) { in tick_nohz_idle_got_tick()
1064 ts->got_idle_tick = 0; in tick_nohz_idle_got_tick()
1091 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_sleep_length() local
1097 ktime_t now = ts->idle_entrytime; in tick_nohz_get_sleep_length()
1100 WARN_ON_ONCE(!ts->inidle); in tick_nohz_get_sleep_length()
1104 if (!can_stop_idle_tick(cpu, ts)) in tick_nohz_get_sleep_length()
1107 next_event = tick_nohz_next_event(ts, cpu); in tick_nohz_get_sleep_length()
1116 hrtimer_next_event_without(&ts->sched_timer)); in tick_nohz_get_sleep_length()
1129 struct tick_sched *ts = tick_get_tick_sched(cpu); in tick_nohz_get_idle_calls_cpu() local
1131 return ts->idle_calls; in tick_nohz_get_idle_calls_cpu()
1141 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_get_idle_calls() local
1143 return ts->idle_calls; in tick_nohz_get_idle_calls()
1146 static void tick_nohz_account_idle_ticks(struct tick_sched *ts) in tick_nohz_account_idle_ticks() argument
1158 ticks = jiffies - ts->idle_jiffies; in tick_nohz_account_idle_ticks()
1167 static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now) in __tick_nohz_idle_restart_tick() argument
1169 tick_nohz_restart_sched_tick(ts, now); in __tick_nohz_idle_restart_tick()
1170 tick_nohz_account_idle_ticks(ts); in __tick_nohz_idle_restart_tick()
1175 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_restart_tick() local
1177 if (ts->tick_stopped) in tick_nohz_idle_restart_tick()
1178 __tick_nohz_idle_restart_tick(ts, ktime_get()); in tick_nohz_idle_restart_tick()
1190 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_idle_exit() local
1196 WARN_ON_ONCE(!ts->inidle); in tick_nohz_idle_exit()
1197 WARN_ON_ONCE(ts->timer_expires_base); in tick_nohz_idle_exit()
1199 ts->inidle = 0; in tick_nohz_idle_exit()
1200 idle_active = ts->idle_active; in tick_nohz_idle_exit()
1201 tick_stopped = ts->tick_stopped; in tick_nohz_idle_exit()
1207 tick_nohz_stop_idle(ts, now); in tick_nohz_idle_exit()
1210 __tick_nohz_idle_restart_tick(ts, now); in tick_nohz_idle_exit()
1220 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_handler() local
1226 tick_sched_do_timer(ts, now); in tick_nohz_handler()
1227 tick_sched_handle(ts, regs); in tick_nohz_handler()
1230 if (unlikely(ts->tick_stopped)) in tick_nohz_handler()
1233 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_nohz_handler()
1234 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_handler()
1237 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) in tick_nohz_activate() argument
1241 ts->nohz_mode = mode; in tick_nohz_activate()
1252 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_switch_to_nohz() local
1262 * Recycle the hrtimer in ts, so we can share the in tick_nohz_switch_to_nohz()
1265 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_nohz_switch_to_nohz()
1269 hrtimer_set_expires(&ts->sched_timer, next); in tick_nohz_switch_to_nohz()
1270 hrtimer_forward_now(&ts->sched_timer, tick_period); in tick_nohz_switch_to_nohz()
1271 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1); in tick_nohz_switch_to_nohz()
1272 tick_nohz_activate(ts, NOHZ_MODE_LOWRES); in tick_nohz_switch_to_nohz()
1277 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_nohz_irq_enter() local
1280 if (!ts->idle_active && !ts->tick_stopped) in tick_nohz_irq_enter()
1283 if (ts->idle_active) in tick_nohz_irq_enter()
1284 tick_nohz_stop_idle(ts, now); in tick_nohz_irq_enter()
1285 if (ts->tick_stopped) in tick_nohz_irq_enter()
1293 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { } in tick_nohz_activate() argument
1316 struct tick_sched *ts = in tick_sched_timer() local
1321 tick_sched_do_timer(ts, now); in tick_sched_timer()
1328 tick_sched_handle(ts, regs); in tick_sched_timer()
1330 ts->next_tick = 0; in tick_sched_timer()
1333 if (unlikely(ts->tick_stopped)) in tick_sched_timer()
1356 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_setup_sched_timer() local
1362 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD); in tick_setup_sched_timer()
1363 ts->sched_timer.function = tick_sched_timer; in tick_setup_sched_timer()
1366 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); in tick_setup_sched_timer()
1373 hrtimer_add_expires_ns(&ts->sched_timer, offset); in tick_setup_sched_timer()
1376 hrtimer_forward(&ts->sched_timer, now, tick_period); in tick_setup_sched_timer()
1377 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD); in tick_setup_sched_timer()
1378 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES); in tick_setup_sched_timer()
1385 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); in tick_cancel_sched_timer() local
1388 if (ts->sched_timer.base) in tick_cancel_sched_timer()
1389 hrtimer_cancel(&ts->sched_timer); in tick_cancel_sched_timer()
1392 memset(ts, 0, sizeof(*ts)); in tick_cancel_sched_timer()
1412 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_oneshot_notify() local
1414 set_bit(0, &ts->check_clocks); in tick_oneshot_notify()
1427 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); in tick_check_oneshot_change() local
1429 if (!test_and_clear_bit(0, &ts->check_clocks)) in tick_check_oneshot_change()
1432 if (ts->nohz_mode != NOHZ_MODE_INACTIVE) in tick_check_oneshot_change()