kernel/sched/clock.c

114 notrace static void __scd_stamp(struct sched_clock_data *scd)
116 	scd->tick_gtod = ktime_get_ns();
117 	scd->tick_raw = sched_clock();
122 	struct sched_clock_data *scd;
126 	 * to disable IRQs in order to get a consistent scd->tick* reading.
129 	scd = this_scd();
133 	__sched_clock_offset = (scd->tick_gtod + __gtod_offset) - (scd->tick_raw);
137 			scd->tick_gtod, __gtod_offset,
138 			scd->tick_raw,  __sched_clock_offset);
157 	struct sched_clock_data *scd;
162 	scd = this_scd();
163 	__scd_stamp(scd);
164 	scd->clock = scd->tick_gtod + __gtod_offset;
169 		per_cpu(sched_clock_data, cpu) = *scd;
173 			scd->tick_gtod, __gtod_offset,
174 			scd->tick_raw,  __sched_clock_offset);
203 	struct sched_clock_data *scd = this_scd();
205 	__scd_stamp(scd);
206 	__gtod_offset = (scd->tick_raw + __sched_clock_offset) - scd->tick_gtod;
264  * update the percpu scd from the raw @now value
269 static __always_inline u64 sched_clock_local(struct sched_clock_data *scd)
276 	delta = now - scd->tick_raw;
280 	old_clock = scd->clock;
283 	 * scd->clock = clamp(scd->tick_gtod + delta,
284 	 *		      max(scd->tick_gtod, scd->clock),
285 	 *		      scd->tick_gtod + TICK_NSEC);
288 	gtod = scd->tick_gtod + __gtod_offset;
296 	if (!raw_try_cmpxchg64(&scd->clock, &old_clock, clock))
327 static notrace u64 sched_clock_remote(struct sched_clock_data *scd)
352 	remote_clock = cmpxchg64(&scd->clock, 0, 0);
355 	 * On 64-bit kernels the read of [my]scd->clock is atomic versus the
361 	remote_clock = scd->clock;
371 		ptr = &scd->clock;
396 	struct sched_clock_data *scd;
406 	scd = cpu_sdc(cpu);
409 		clock = sched_clock_remote(scd);
411 		clock = sched_clock_local(scd);
420 	struct sched_clock_data *scd;
430 	scd = this_scd();
431 	__scd_stamp(scd);
432 	sched_clock_local(scd);