Lines Matching refs:tk
70 static inline bool tk_is_aux(const struct timekeeper *tk) in tk_is_aux() argument
72 return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST; in tk_is_aux()
80 static inline bool tk_is_aux(const struct timekeeper *tk) in tk_is_aux() argument
86 static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs) in tk_update_aux_offs() argument
88 tk->offs_aux = offs; in tk_update_aux_offs()
89 tk->monotonic_to_aux = ktime_to_timespec64(offs); in tk_update_aux_offs()
190 static inline void tk_normalize_xtime(struct timekeeper *tk) in tk_normalize_xtime() argument
192 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { in tk_normalize_xtime()
193 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in tk_normalize_xtime()
194 tk->xtime_sec++; in tk_normalize_xtime()
196 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { in tk_normalize_xtime()
197 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in tk_normalize_xtime()
198 tk->raw_sec++; in tk_normalize_xtime()
202 static inline struct timespec64 tk_xtime(const struct timekeeper *tk) in tk_xtime() argument
206 ts.tv_sec = tk->xtime_sec; in tk_xtime()
207 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_xtime()
211 static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk) in tk_xtime_coarse() argument
215 ts.tv_sec = tk->xtime_sec; in tk_xtime_coarse()
216 ts.tv_nsec = tk->coarse_nsec; in tk_xtime_coarse()
229 static inline void tk_update_coarse_nsecs(struct timekeeper *tk) in tk_update_coarse_nsecs() argument
231 tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; in tk_update_coarse_nsecs()
234 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) in tk_set_xtime() argument
236 tk->xtime_sec = ts->tv_sec; in tk_set_xtime()
237 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_set_xtime()
238 tk_update_coarse_nsecs(tk); in tk_set_xtime()
241 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) in tk_xtime_add() argument
243 tk->xtime_sec += ts->tv_sec; in tk_xtime_add()
244 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_xtime_add()
245 tk_normalize_xtime(tk); in tk_xtime_add()
246 tk_update_coarse_nsecs(tk); in tk_xtime_add()
249 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) in tk_set_wall_to_mono() argument
257 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, in tk_set_wall_to_mono()
258 -tk->wall_to_monotonic.tv_nsec); in tk_set_wall_to_mono()
259 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); in tk_set_wall_to_mono()
260 tk->wall_to_monotonic = wtm; in tk_set_wall_to_mono()
263 WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp)); in tk_set_wall_to_mono()
264 WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0))); in tk_set_wall_to_mono()
267 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) in tk_update_sleep_time() argument
270 WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta)); in tk_update_sleep_time()
275 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); in tk_update_sleep_time()
309 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) in tk_setup_internals() argument
315 ++tk->cs_was_changed_seq; in tk_setup_internals()
316 old_clock = tk->tkr_mono.clock; in tk_setup_internals()
317 tk->tkr_mono.clock = clock; in tk_setup_internals()
318 tk->tkr_mono.mask = clock->mask; in tk_setup_internals()
319 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); in tk_setup_internals()
321 tk->tkr_raw.clock = clock; in tk_setup_internals()
322 tk->tkr_raw.mask = clock->mask; in tk_setup_internals()
323 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; in tk_setup_internals()
335 tk->cycle_interval = interval; in tk_setup_internals()
338 tk->xtime_interval = interval * clock->mult; in tk_setup_internals()
339 tk->xtime_remainder = ntpinterval - tk->xtime_interval; in tk_setup_internals()
340 tk->raw_interval = interval * clock->mult; in tk_setup_internals()
346 tk->tkr_mono.xtime_nsec >>= -shift_change; in tk_setup_internals()
347 tk->tkr_raw.xtime_nsec >>= -shift_change; in tk_setup_internals()
349 tk->tkr_mono.xtime_nsec <<= shift_change; in tk_setup_internals()
350 tk->tkr_raw.xtime_nsec <<= shift_change; in tk_setup_internals()
354 tk->tkr_mono.shift = clock->shift; in tk_setup_internals()
355 tk->tkr_raw.shift = clock->shift; in tk_setup_internals()
357 tk->ntp_error = 0; in tk_setup_internals()
358 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; in tk_setup_internals()
359 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; in tk_setup_internals()
366 tk->tkr_mono.mult = clock->mult; in tk_setup_internals()
367 tk->tkr_raw.mult = clock->mult; in tk_setup_internals()
368 tk->ntp_err_mult = 0; in tk_setup_internals()
369 tk->skip_second_overflow = 0; in tk_setup_internals()
534 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_boot_fast_ns() local
536 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot))); in ktime_get_boot_fast_ns()
551 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_tai_fast_ns() local
553 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai))); in ktime_get_tai_fast_ns()
590 static void halt_fast_timekeeper(const struct timekeeper *tk) in halt_fast_timekeeper() argument
593 const struct tk_read_base *tkr = &tk->tkr_mono; in halt_fast_timekeeper()
598 tkr_dummy.base_real = tkr->base + tk->offs_real; in halt_fast_timekeeper()
601 tkr = &tk->tkr_raw; in halt_fast_timekeeper()
609 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) in update_pvclock_gtod() argument
611 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); in update_pvclock_gtod()
620 struct timekeeper *tk = &tk_core.timekeeper; in pvclock_gtod_register_notifier() local
625 update_pvclock_gtod(tk, true); in pvclock_gtod_register_notifier()
646 static inline void tk_update_leap_state(struct timekeeper *tk) in tk_update_leap_state() argument
648 tk->next_leap_ktime = ntp_get_next_leap(tk->id); in tk_update_leap_state()
649 if (tk->next_leap_ktime != KTIME_MAX) in tk_update_leap_state()
651 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); in tk_update_leap_state()
669 static inline void tk_update_ktime_data(struct timekeeper *tk) in tk_update_ktime_data() argument
681 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); in tk_update_ktime_data()
682 nsec = (u32) tk->wall_to_monotonic.tv_nsec; in tk_update_ktime_data()
683 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); in tk_update_ktime_data()
690 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_update_ktime_data()
693 tk->ktime_sec = seconds; in tk_update_ktime_data()
696 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); in tk_update_ktime_data()
710 struct timekeeper *tk = &tkd->shadow_timekeeper; in timekeeping_update_from_shadow() local
724 tk->ntp_error = 0; in timekeeping_update_from_shadow()
725 ntp_clear(tk->id); in timekeeping_update_from_shadow()
728 tk_update_leap_state(tk); in timekeeping_update_from_shadow()
729 tk_update_ktime_data(tk); in timekeeping_update_from_shadow()
730 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; in timekeeping_update_from_shadow()
732 if (tk->id == TIMEKEEPER_CORE) { in timekeeping_update_from_shadow()
733 update_vsyscall(tk); in timekeeping_update_from_shadow()
734 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); in timekeeping_update_from_shadow()
736 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); in timekeeping_update_from_shadow()
737 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); in timekeeping_update_from_shadow()
738 } else if (tk_is_aux(tk)) { in timekeeping_update_from_shadow()
739 vdso_time_update_aux(tk); in timekeeping_update_from_shadow()
743 tk->clock_was_set_seq++; in timekeeping_update_from_shadow()
753 memcpy(&tkd->timekeeper, tk, sizeof(*tk)); in timekeeping_update_from_shadow()
765 static void timekeeping_forward_now(struct timekeeper *tk) in timekeeping_forward_now() argument
769 cycle_now = tk_clock_read(&tk->tkr_mono); in timekeeping_forward_now()
770 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask, in timekeeping_forward_now()
771 tk->tkr_mono.clock->max_raw_delta); in timekeeping_forward_now()
772 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_forward_now()
773 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_forward_now()
776 u64 max = tk->tkr_mono.clock->max_cycles; in timekeeping_forward_now()
779 tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult; in timekeeping_forward_now()
780 tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult; in timekeeping_forward_now()
781 tk_normalize_xtime(tk); in timekeeping_forward_now()
784 tk_update_coarse_nsecs(tk); in timekeeping_forward_now()
795 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_ts64() local
804 ts->tv_sec = tk->xtime_sec; in ktime_get_real_ts64()
805 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_real_ts64()
816 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get() local
825 base = tk->tkr_mono.base; in ktime_get()
826 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get()
836 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_resolution_ns() local
844 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; in ktime_get_resolution_ns()
859 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_with_offset() local
868 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_with_offset()
869 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_with_offset()
880 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_with_offset() local
889 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_coarse_with_offset()
890 nsecs = tk->coarse_nsec; in ktime_get_coarse_with_offset()
931 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw() local
938 base = tk->tkr_raw.base; in ktime_get_raw()
939 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw()
957 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_ts64() local
966 ts->tv_sec = tk->xtime_sec; in ktime_get_ts64()
967 nsec = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_ts64()
968 tomono = tk->wall_to_monotonic; in ktime_get_ts64()
989 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_seconds() local
992 return tk->ktime_sec; in ktime_get_seconds()
1008 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_seconds() local
1013 return tk->xtime_sec; in ktime_get_real_seconds()
1017 seconds = tk->xtime_sec; in ktime_get_real_seconds()
1037 struct timekeeper *tk = &tk_core.timekeeper; in __ktime_get_real_seconds() local
1039 return tk->xtime_sec; in __ktime_get_real_seconds()
1048 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_snapshot() local
1061 now = tk_clock_read(&tk->tkr_mono); in ktime_get_snapshot()
1062 systime_snapshot->cs_id = tk->tkr_mono.clock->id; in ktime_get_snapshot()
1063 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; in ktime_get_snapshot()
1064 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; in ktime_get_snapshot()
1065 base_real = ktime_add(tk->tkr_mono.base, in ktime_get_snapshot()
1067 base_boot = ktime_add(tk->tkr_mono.base, in ktime_get_snapshot()
1069 base_raw = tk->tkr_raw.base; in ktime_get_snapshot()
1070 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now); in ktime_get_snapshot()
1071 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); in ktime_get_snapshot()
1121 struct timekeeper *tk = &tk_core.timekeeper; in adjust_historical_crosststamp() local
1155 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); in adjust_historical_crosststamp()
1273 struct timekeeper *tk = &tk_core.timekeeper; in ktime_real_to_base_clock() local
1279 if ((u64)treal < tk->tkr_mono.base_real) in ktime_real_to_base_clock()
1281 delta = (u64)treal - tk->tkr_mono.base_real; in ktime_real_to_base_clock()
1284 *cycles = tk->tkr_mono.cycle_last + delta; in ktime_real_to_base_clock()
1313 struct timekeeper *tk = &tk_core.timekeeper; in get_device_system_crosststamp() local
1347 now = tk_clock_read(&tk->tkr_mono); in get_device_system_crosststamp()
1348 interval_start = tk->tkr_mono.cycle_last; in get_device_system_crosststamp()
1350 clock_was_set_seq = tk->clock_was_set_seq; in get_device_system_crosststamp()
1351 cs_was_changed_seq = tk->cs_was_changed_seq; in get_device_system_crosststamp()
1358 base_real = ktime_add(tk->tkr_mono.base, in get_device_system_crosststamp()
1360 base_raw = tk->tkr_raw.base; in get_device_system_crosststamp()
1362 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); in get_device_system_crosststamp()
1363 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles); in get_device_system_crosststamp()
1468 static inline bool timekeeper_is_core_tk(struct timekeeper *tk) in timekeeper_is_core_tk() argument
1470 return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE; in timekeeper_is_core_tk()
1572 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) in __timekeeping_set_tai_offset() argument
1574 tk->tai_offset = tai_offset; in __timekeeping_set_tai_offset()
1575 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); in __timekeeping_set_tai_offset()
1630 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_notify() local
1632 if (tk->tkr_mono.clock == clock) in timekeeping_notify()
1636 return tk->tkr_mono.clock == clock ? 0 : -1; in timekeeping_notify()
1647 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw_ts64() local
1653 ts->tv_sec = tk->raw_sec; in ktime_get_raw_ts64()
1654 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw_ts64()
1702 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_valid_for_hres() local
1709 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; in timekeeping_valid_for_hres()
1721 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_max_deferment() local
1728 ret = tk->tkr_mono.clock->max_idle_ns; in timekeeping_max_deferment()
1856 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, in __timekeeping_inject_sleeptime() argument
1865 tk_xtime_add(tk, delta); in __timekeeping_inject_sleeptime()
1866 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); in __timekeeping_inject_sleeptime()
1867 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); in __timekeeping_inject_sleeptime()
2094 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, in timekeeping_apply_adjustment() argument
2098 s64 interval = tk->cycle_interval; in timekeeping_apply_adjustment()
2157 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { in timekeeping_apply_adjustment()
2163 tk->tkr_mono.mult += mult_adj; in timekeeping_apply_adjustment()
2164 tk->xtime_interval += interval; in timekeeping_apply_adjustment()
2165 tk->tkr_mono.xtime_nsec -= offset; in timekeeping_apply_adjustment()
2172 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) in timekeeping_adjust() argument
2174 u64 ntp_tl = ntp_tick_length(tk->id); in timekeeping_adjust()
2181 if (likely(tk->ntp_tick == ntp_tl)) { in timekeeping_adjust()
2182 mult = tk->tkr_mono.mult - tk->ntp_err_mult; in timekeeping_adjust()
2184 tk->ntp_tick = ntp_tl; in timekeeping_adjust()
2185 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - in timekeeping_adjust()
2186 tk->xtime_remainder, tk->cycle_interval); in timekeeping_adjust()
2195 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; in timekeeping_adjust()
2196 mult += tk->ntp_err_mult; in timekeeping_adjust()
2198 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); in timekeeping_adjust()
2200 if (unlikely(tk->tkr_mono.clock->maxadj && in timekeeping_adjust()
2201 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) in timekeeping_adjust()
2202 > tk->tkr_mono.clock->maxadj))) { in timekeeping_adjust()
2205 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, in timekeeping_adjust()
2206 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); in timekeeping_adjust()
2219 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { in timekeeping_adjust()
2220 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << in timekeeping_adjust()
2221 tk->tkr_mono.shift; in timekeeping_adjust()
2222 tk->xtime_sec--; in timekeeping_adjust()
2223 tk->skip_second_overflow = 1; in timekeeping_adjust()
2234 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) in accumulate_nsecs_to_secs() argument
2236 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in accumulate_nsecs_to_secs()
2239 while (tk->tkr_mono.xtime_nsec >= nsecps) { in accumulate_nsecs_to_secs()
2242 tk->tkr_mono.xtime_nsec -= nsecps; in accumulate_nsecs_to_secs()
2243 tk->xtime_sec++; in accumulate_nsecs_to_secs()
2249 if (unlikely(tk->skip_second_overflow)) { in accumulate_nsecs_to_secs()
2250 tk->skip_second_overflow = 0; in accumulate_nsecs_to_secs()
2255 leap = second_overflow(tk->id, tk->xtime_sec); in accumulate_nsecs_to_secs()
2259 tk->xtime_sec += leap; in accumulate_nsecs_to_secs()
2263 tk_set_wall_to_mono(tk, in accumulate_nsecs_to_secs()
2264 timespec64_sub(tk->wall_to_monotonic, ts)); in accumulate_nsecs_to_secs()
2266 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); in accumulate_nsecs_to_secs()
2283 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, in logarithmic_accumulation() argument
2286 u64 interval = tk->cycle_interval << shift; in logarithmic_accumulation()
2295 tk->tkr_mono.cycle_last += interval; in logarithmic_accumulation()
2296 tk->tkr_raw.cycle_last += interval; in logarithmic_accumulation()
2298 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; in logarithmic_accumulation()
2299 *clock_set |= accumulate_nsecs_to_secs(tk); in logarithmic_accumulation()
2302 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; in logarithmic_accumulation()
2303 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; in logarithmic_accumulation()
2304 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { in logarithmic_accumulation()
2305 tk->tkr_raw.xtime_nsec -= snsec_per_sec; in logarithmic_accumulation()
2306 tk->raw_sec++; in logarithmic_accumulation()
2310 tk->ntp_error += tk->ntp_tick << shift; in logarithmic_accumulation()
2311 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << in logarithmic_accumulation()
2312 (tk->ntp_error_shift + shift); in logarithmic_accumulation()
2323 struct timekeeper *tk = &tkd->shadow_timekeeper; in __timekeeping_advance() local
2333 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), in __timekeeping_advance()
2334 tk->tkr_mono.cycle_last, tk->tkr_mono.mask, in __timekeeping_advance()
2335 tk->tkr_mono.clock->max_raw_delta); in __timekeeping_advance()
2349 shift = ilog2(offset) - ilog2(tk->cycle_interval); in __timekeeping_advance()
2352 maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1; in __timekeeping_advance()
2354 while (offset >= tk->cycle_interval) { in __timekeeping_advance()
2355 offset = logarithmic_accumulation(tk, offset, shift, &clock_set); in __timekeeping_advance()
2356 if (offset < tk->cycle_interval<<shift) in __timekeeping_advance()
2361 timekeeping_adjust(tk, offset); in __timekeeping_advance()
2367 clock_set |= accumulate_nsecs_to_secs(tk); in __timekeeping_advance()
2375 tk_update_coarse_nsecs(tk); in __timekeeping_advance()
2413 struct timekeeper *tk = &tk_core.timekeeper; in getboottime64() local
2414 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); in getboottime64()
2422 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_real_ts64() local
2428 *ts = tk_xtime_coarse(tk); in ktime_get_coarse_real_ts64()
2444 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_real_ts64_mg() local
2451 *ts = tk_xtime_coarse(tk); in ktime_get_coarse_real_ts64_mg()
2482 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_ts64_mg() local
2491 ts->tv_sec = tk->xtime_sec; in ktime_get_real_ts64_mg()
2492 mono = tk->tkr_mono.base; in ktime_get_real_ts64_mg()
2493 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_real_ts64_mg()
2523 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_coarse_ts64() local
2530 now = tk_xtime_coarse(tk); in ktime_get_coarse_ts64()
2531 mono = tk->wall_to_monotonic; in ktime_get_coarse_ts64()
2564 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_update_offsets_now() local
2572 base = tk->tkr_mono.base; in ktime_get_update_offsets_now()
2573 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_update_offsets_now()
2576 if (*cwsseq != tk->clock_was_set_seq) { in ktime_get_update_offsets_now()
2577 *cwsseq = tk->clock_was_set_seq; in ktime_get_update_offsets_now()
2578 *offs_real = tk->offs_real; in ktime_get_update_offsets_now()
2579 *offs_boot = tk->offs_boot; in ktime_get_update_offsets_now()
2580 *offs_tai = tk->offs_tai; in ktime_get_update_offsets_now()
2584 if (unlikely(base >= tk->next_leap_ktime)) in ktime_get_update_offsets_now()
2585 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); in ktime_get_update_offsets_now()